Forecasting app: hybrid port to HNF stack

Python FastAPI ML backend kept intact; auth replaced with central hnf_session cookie verification. Frontend rebuilt on React 18 + TS + Vite with stack design system, Plotly charts retained. Shared Postgres via DATABASE_URL; schema applied on startup.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitignore

@@ -0,0 +1,13 @@
+node_modules/
+dist/
+__pycache__/
+*.pyc
+*.pyo
+.env
+.env.local
+.DS_Store
+*.egg-info/
+.eggs/
+build/
+.venv/
+venv/

backend/Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libpq-dev \
+    postgresql-client \
+    g++ \
+    make \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Playwright + Chromium for Booking.com scraper
+RUN playwright install chromium --with-deps
+
+# CmdStan for Prophet
+RUN python -m cmdstanpy.install_cmdstan --cores 2
+
+# Fix Prophet cmdstan path symlink
+RUN CMDSTAN_VERSION=$(ls /root/.cmdstan/ | head -1) && \
+    rm -rf /usr/local/lib/python3.11/site-packages/prophet/stan_model/cmdstan-* && \
+    ln -s /root/.cmdstan/$CMDSTAN_VERSION /usr/local/lib/python3.11/site-packages/prophet/stan_model/cmdstan-2.31.0
+
+COPY . .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/api/__init__.py

@@ -0,0 +1 @@
+# API routers

backend/api/accuracy.py

@@ -0,0 +1,457 @@
+"""
+Accuracy tracking API endpoints
+"""
+from datetime import date, timedelta
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+@router.get("/summary")
+async def get_accuracy_summary(
+    from_date: date = Query(..., description="Start date"),
+    to_date: date = Query(..., description="End date"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get model accuracy comparison over date range.
+    Returns MAE, RMSE, MAPE for each model.
+    """
+    query = """
+        SELECT
+            metric_type,
+            COUNT(*) as sample_count,
+            -- Prophet metrics
+            AVG(ABS(prophet_error)) as prophet_mae,
+            SQRT(AVG(prophet_error * prophet_error)) as prophet_rmse,
+            AVG(ABS(prophet_pct_error)) as prophet_mape,
+            -- XGBoost metrics
+            AVG(ABS(xgboost_error)) as xgboost_mae,
+            SQRT(AVG(xgboost_error * xgboost_error)) as xgboost_rmse,
+            AVG(ABS(xgboost_pct_error)) as xgboost_mape,
+            -- CatBoost metrics
+            AVG(ABS(catboost_error)) as catboost_mae,
+            SQRT(AVG(catboost_error * catboost_error)) as catboost_rmse,
+            AVG(ABS(catboost_pct_error)) as catboost_mape,
+            -- Pickup metrics
+            AVG(ABS(pickup_error)) as pickup_mae,
+            SQRT(AVG(pickup_error * pickup_error)) as pickup_rmse,
+            AVG(ABS(pickup_pct_error)) as pickup_mape,
+            -- Best model distribution
+            SUM(CASE WHEN best_model = 'prophet' THEN 1 ELSE 0 END) as prophet_wins,
+            SUM(CASE WHEN best_model = 'xgboost' THEN 1 ELSE 0 END) as xgboost_wins,
+            SUM(CASE WHEN best_model = 'catboost' THEN 1 ELSE 0 END) as catboost_wins,
+            SUM(CASE WHEN best_model = 'pickup' THEN 1 ELSE 0 END) as pickup_wins
+        FROM actual_vs_forecast
+        WHERE date BETWEEN :from_date AND :to_date
+            AND actual_value IS NOT NULL
+        GROUP BY metric_type
+        ORDER BY metric_type
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "metric_type": row.metric_type,
+            "sample_count": row.sample_count,
+            "prophet": {
+                "mae": round(float(row.prophet_mae), 2) if row.prophet_mae else None,
+                "rmse": round(float(row.prophet_rmse), 2) if row.prophet_rmse else None,
+                "mape": round(float(row.prophet_mape), 2) if row.prophet_mape else None,
+                "wins": row.prophet_wins
+            },
+            "xgboost": {
+                "mae": round(float(row.xgboost_mae), 2) if row.xgboost_mae else None,
+                "rmse": round(float(row.xgboost_rmse), 2) if row.xgboost_rmse else None,
+                "mape": round(float(row.xgboost_mape), 2) if row.xgboost_mape else None,
+                "wins": row.xgboost_wins
+            },
+            "catboost": {
+                "mae": round(float(row.catboost_mae), 2) if row.catboost_mae else None,
+                "rmse": round(float(row.catboost_rmse), 2) if row.catboost_rmse else None,
+                "mape": round(float(row.catboost_mape), 2) if row.catboost_mape else None,
+                "wins": row.catboost_wins
+            },
+            "pickup": {
+                "mae": round(float(row.pickup_mae), 2) if row.pickup_mae else None,
+                "rmse": round(float(row.pickup_rmse), 2) if row.pickup_rmse else None,
+                "mape": round(float(row.pickup_mape), 2) if row.pickup_mape else None,
+                "wins": row.pickup_wins
+            }
+        }
+        for row in rows
+    ]
+
+
+@router.get("/model-weights")
+async def get_model_weights(
+    metric_code: Optional[str] = Query(None, description="Metric code (if None, returns all metrics)"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get MAPE-based model weights used for blending.
+    Shows the actual MAPE scores from backtest data and calculated weights.
+    This is what the blended_tuned_weighted service uses for weighting.
+    """
+    # Map metric codes to forecast_snapshots metric codes
+    metric_map = {
+        'hotel_occupancy_pct': 'occupancy',
+        'hotel_room_nights': 'rooms',
+        'hotel_guests': 'guests',
+        'hotel_arr': 'arr',
+        'ave_guest_rate': 'ave_guest_rate',
+        'net_accom': 'net_accom',
+        'net_dry': 'net_dry',
+        'net_wet': 'net_wet',
+        'total_rev': 'total_rev',
+    }
+
+    # Pace metrics use pickup model
+    pace_metrics = ['hotel_occupancy_pct', 'hotel_room_nights']
+
+    # If specific metric requested, process just that one
+    metrics_to_process = [metric_code] if metric_code else list(metric_map.keys())
+
+    results = []
+    for metric in metrics_to_process:
+        snapshot_metric = metric_map.get(metric, metric)
+        is_pace_metric = metric in pace_metrics
+
+        # Query MAPE from forecast_snapshots
+        models_to_query = ['prophet', 'xgboost', 'catboost']
+        if is_pace_metric:
+            models_to_query.append('pickup')
+
+        mape_scores = {}
+        sample_counts = {}
+        for model in models_to_query:
+            query = text("""
+                SELECT
+                    AVG(ABS((forecast_value - actual_value) / NULLIF(actual_value, 0)) * 100) as mape,
+                    COUNT(*) as sample_count
+                FROM forecast_snapshots
+                WHERE actual_value IS NOT NULL
+                    AND actual_value != 0
+                    AND forecast_value IS NOT NULL
+                    AND metric_code = :metric_code
+                    AND model = :model
+            """)
+            result = await db.execute(query, {"metric_code": snapshot_metric, "model": model})
+            row = result.fetchone()
+
+            if row and row.mape is not None:
+                mape_scores[model] = float(row.mape)
+                sample_counts[model] = int(row.sample_count)
+            else:
+                mape_scores[model] = None
+                sample_counts[model] = 0
+
+        # Calculate inverse-MAPE weights (same logic as blended_tuned_weighted)
+        valid_mapes = {k: v for k, v in mape_scores.items() if v is not None}
+
+        if valid_mapes:
+            # Calculate weights: lower MAPE = higher weight
+            weights = {model: 1.0 / max(mape, 0.1) for model, mape in valid_mapes.items()}
+            weight_sum = sum(weights.values())
+            normalized_weights = {k: v / weight_sum for k, v in weights.items()}
+        else:
+            # Fall back to equal weights
+            normalized_weights = {model: 1.0 / len(models_to_query) for model in models_to_query}
+
+        # Build response
+        model_data = {}
+        for model in models_to_query:
+            model_data[model] = {
+                "mape": round(mape_scores.get(model), 2) if mape_scores.get(model) is not None else None,
+                "weight": round(normalized_weights.get(model, 0), 4),
+                "sample_count": sample_counts.get(model, 0)
+            }
+
+        results.append({
+            "metric_code": metric,
+            "snapshot_metric": snapshot_metric,
+            "is_pace_metric": is_pace_metric,
+            "models": model_data,
+            "total_samples": sum(sample_counts.values())
+        })
+
+    return results
+
+
+@router.get("/by-model")
+async def get_accuracy_by_model(
+    model: str = Query(..., description="Model: prophet, xgboost, catboost, pickup"),
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    metric_type: Optional[str] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get detailed accuracy for a specific model.
+    """
+    column_map = {
+        "prophet": ("prophet_forecast", "prophet_error", "prophet_pct_error"),
+        "xgboost": ("xgboost_forecast", "xgboost_error", "xgboost_pct_error"),
+        "catboost": ("catboost_forecast", "catboost_error", "catboost_pct_error"),
+        "pickup": ("pickup_forecast", "pickup_error", "pickup_pct_error")
+    }
+
+    if model not in column_map:
+        raise ValueError(f"Invalid model: {model}")
+
+    forecast_col, error_col, pct_error_col = column_map[model]
+
+    query = f"""
+        SELECT
+            date,
+            metric_type,
+            actual_value,
+            {forecast_col} as forecast,
+            {error_col} as error,
+            {pct_error_col} as pct_error,
+            best_model
+        FROM actual_vs_forecast
+        WHERE date BETWEEN :from_date AND :to_date
+            AND actual_value IS NOT NULL
+    """
+    params = {"from_date": from_date, "to_date": to_date}
+
+    if metric_type:
+        query += " AND metric_type = :metric_type"
+        params["metric_type"] = metric_type
+
+    query += " ORDER BY date, metric_type"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.date,
+            "metric_type": row.metric_type,
+            "actual": float(row.actual_value),
+            "forecast": float(row.forecast) if row.forecast else None,
+            "error": float(row.error) if row.error else None,
+            "pct_error": float(row.pct_error) if row.pct_error else None,
+            "was_best": row.best_model == model
+        }
+        for row in rows
+    ]
+
+
+@router.get("/best-model")
+async def get_best_model_analysis(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Analyze which model performs best by metric type and time period.
+    """
+    query = """
+        WITH model_performance AS (
+            SELECT
+                metric_type,
+                DATE_TRUNC('week', date) as week,
+                best_model,
+                COUNT(*) as count
+            FROM actual_vs_forecast
+            WHERE date BETWEEN :from_date AND :to_date
+                AND actual_value IS NOT NULL
+            GROUP BY metric_type, DATE_TRUNC('week', date), best_model
+        )
+        SELECT
+            metric_type,
+            week,
+            best_model,
+            count,
+            ROUND(count * 100.0 / SUM(count) OVER (PARTITION BY metric_type, week), 1) as pct
+        FROM model_performance
+        ORDER BY metric_type, week, count DESC
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "metric_type": row.metric_type,
+            "week": row.week,
+            "best_model": row.best_model,
+            "count": row.count,
+            "percentage": float(row.pct)
+        }
+        for row in rows
+    ]
+
+
+@router.get("/by-lead-time")
+async def get_accuracy_by_lead_time(
+    from_date: Optional[date] = Query(None),
+    to_date: Optional[date] = Query(None),
+    metric_code: Optional[str] = Query(None, description="Filter by metric code"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get accuracy aggregated by lead time brackets from backtest data.
+    Returns MAPE for each model at different lead times (7, 14, 28, 60, 90 days).
+    """
+    if from_date is None:
+        from_date = date.today() - timedelta(days=365)
+    if to_date is None:
+        to_date = date.today()
+
+    # Define lead time brackets
+    brackets = [
+        (0, 7, "1 week"),
+        (8, 14, "2 weeks"),
+        (15, 28, "1 month"),
+        (29, 60, "2 months"),
+        (61, 90, "3 months"),
+        (91, 180, "6 months"),
+        (181, 365, "1 year")
+    ]
+
+    query = """
+        SELECT
+            CASE
+                WHEN days_out <= 7 THEN '1 week'
+                WHEN days_out <= 14 THEN '2 weeks'
+                WHEN days_out <= 28 THEN '1 month'
+                WHEN days_out <= 60 THEN '2 months'
+                WHEN days_out <= 90 THEN '3 months'
+                WHEN days_out <= 180 THEN '6 months'
+                ELSE '1 year'
+            END as lead_time_label,
+            CASE
+                WHEN days_out <= 7 THEN 1
+                WHEN days_out <= 14 THEN 2
+                WHEN days_out <= 28 THEN 3
+                WHEN days_out <= 60 THEN 4
+                WHEN days_out <= 90 THEN 5
+                WHEN days_out <= 180 THEN 6
+                ELSE 7
+            END as sort_order,
+            model,
+            metric_code,
+            AVG(ABS((forecast_value - actual_value) / NULLIF(actual_value, 0) * 100)) as mape,
+            AVG(ABS(forecast_value - actual_value)) as mae,
+            COUNT(*) as sample_count
+        FROM forecast_snapshots
+        WHERE target_date BETWEEN :from_date AND :to_date
+            AND actual_value IS NOT NULL
+    """
+    params = {"from_date": from_date, "to_date": to_date}
+
+    if metric_code:
+        query += " AND metric_code = :metric_code"
+        params["metric_code"] = metric_code
+
+    query += """
+        GROUP BY
+            CASE
+                WHEN days_out <= 7 THEN '1 week'
+                WHEN days_out <= 14 THEN '2 weeks'
+                WHEN days_out <= 28 THEN '1 month'
+                WHEN days_out <= 60 THEN '2 months'
+                WHEN days_out <= 90 THEN '3 months'
+                WHEN days_out <= 180 THEN '6 months'
+                ELSE '1 year'
+            END,
+            CASE
+                WHEN days_out <= 7 THEN 1
+                WHEN days_out <= 14 THEN 2
+                WHEN days_out <= 28 THEN 3
+                WHEN days_out <= 60 THEN 4
+                WHEN days_out <= 90 THEN 5
+                WHEN days_out <= 180 THEN 6
+                ELSE 7
+            END,
+            model,
+            metric_code
+        ORDER BY sort_order, model
+    """
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "lead_time": row.lead_time_label,
+            "model": row.model,
+            "metric_code": row.metric_code,
+            "mape": round(float(row.mape), 2) if row.mape else None,
+            "mae": round(float(row.mae), 2) if row.mae else None,
+            "sample_count": row.sample_count
+        }
+        for row in rows
+    ]
+
+
+@router.get("/by-horizon")
+async def get_accuracy_by_horizon(
+    horizon: int = Query(..., description="Lead time in days (7, 14, 28)"),
+    from_date: Optional[date] = Query(None),
+    to_date: Optional[date] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get accuracy at different lead times from backtest data.
+    Shows how forecast accuracy degrades as horizon increases.
+    Uses forecast_snapshots table populated by backtests.
+    """
+    if from_date is None:
+        from_date = date.today() - timedelta(days=90)
+    if to_date is None:
+        to_date = date.today()
+
+    # Query forecast_snapshots table (populated by backtests)
+    query = """
+        SELECT
+            metric_code as forecast_type,
+            days_out as horizon_days,
+            model as model_type,
+            AVG(ABS(forecast_value - actual_value)) as mae,
+            AVG(ABS((forecast_value - actual_value) / NULLIF(actual_value, 0) * 100)) as mape,
+            COUNT(*) as sample_count
+        FROM forecast_snapshots
+        WHERE target_date BETWEEN :from_date AND :to_date
+            AND days_out = :horizon
+            AND actual_value IS NOT NULL
+        GROUP BY metric_code, days_out, model
+        ORDER BY metric_code, model
+    """
+
+    result = await db.execute(text(query), {
+        "from_date": from_date,
+        "to_date": to_date,
+        "horizon": horizon
+    })
+    rows = result.fetchall()
+
+    return [
+        {
+            "forecast_type": row.forecast_type,
+            "horizon_days": row.horizon_days,
+            "model_type": row.model_type,
+            "mae": round(float(row.mae), 2) if row.mae else None,
+            "mape": round(float(row.mape), 2) if row.mape else None,
+            "sample_count": row.sample_count
+        }
+        for row in rows
+    ]

backend/api/ai_insights.py

@@ -0,0 +1,177 @@
+"""
+AI Insights API Endpoints
+
+Serves pre-computed daily insights to the dashboard and allows manual generation.
+"""
+import logging
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from database import get_db
+from auth import get_current_user
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+# Rate limit: minimum minutes between manual generations
+MANUAL_RATE_LIMIT_MINUTES = 5
+
+
+@router.get("/latest")
+async def get_latest_insight(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get the most recent AI insight for the dashboard card."""
+    result = await db.execute(
+        text("""
+            SELECT id, generated_at, insight_type, content, model,
+                   input_tokens, output_tokens, triggered_by
+            FROM ai_insights
+            ORDER BY generated_at DESC
+            LIMIT 1
+        """)
+    )
+    row = result.fetchone()
+    if not row:
+        return None
+
+    return {
+        "id": row.id,
+        "generated_at": row.generated_at.isoformat(),
+        "insight_type": row.insight_type,
+        "content": row.content,
+        "model": row.model,
+        "input_tokens": row.input_tokens,
+        "output_tokens": row.output_tokens,
+        "triggered_by": row.triggered_by,
+    }
+
+
+@router.get("/history")
+async def get_insight_history(
+    limit: int = Query(default=20, ge=1, le=100),
+    offset: int = Query(default=0, ge=0),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get historical AI insights with pagination."""
+    result = await db.execute(
+        text("""
+            SELECT id, generated_at, insight_type, content, model,
+                   input_tokens, output_tokens, triggered_by
+            FROM ai_insights
+            ORDER BY generated_at DESC
+            LIMIT :limit OFFSET :offset
+        """),
+        {"limit": limit, "offset": offset}
+    )
+
+    count_result = await db.execute(text("SELECT COUNT(*) FROM ai_insights"))
+    total = count_result.scalar()
+
+    insights = []
+    for row in result.fetchall():
+        insights.append({
+            "id": row.id,
+            "generated_at": row.generated_at.isoformat(),
+            "insight_type": row.insight_type,
+            "content": row.content,
+            "model": row.model,
+            "input_tokens": row.input_tokens,
+            "output_tokens": row.output_tokens,
+            "triggered_by": row.triggered_by,
+        })
+
+    return {"insights": insights, "total": total}
+
+
+@router.get("/usage")
+async def get_usage_stats(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get token usage statistics for the current month."""
+    result = await db.execute(
+        text("""
+            SELECT
+                COUNT(*) as generation_count,
+                COALESCE(SUM(input_tokens), 0) as total_input_tokens,
+                COALESCE(SUM(output_tokens), 0) as total_output_tokens
+            FROM ai_insights
+            WHERE generated_at >= DATE_TRUNC('month', CURRENT_DATE)
+        """)
+    )
+    row = result.fetchone()
+
+    # Also get today's usage for budget display
+    today_result = await db.execute(
+        text("""
+            SELECT COALESCE(SUM(input_tokens + output_tokens), 0) as today_total
+            FROM ai_insights
+            WHERE generated_at >= CURRENT_DATE
+        """)
+    )
+    today_row = today_result.fetchone()
+
+    return {
+        "month": {
+            "generations": row.generation_count,
+            "input_tokens": row.total_input_tokens,
+            "output_tokens": row.total_output_tokens,
+            "total_tokens": row.total_input_tokens + row.total_output_tokens,
+        },
+        "today_tokens": today_row.today_total if today_row else 0,
+    }
+
+
+@router.post("/generate")
+async def generate_insight_manual(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Manually trigger AI insight generation.
+    Rate-limited to prevent accidental spam.
+    """
+    # Check rate limit
+    result = await db.execute(
+        text("""
+            SELECT generated_at FROM ai_insights
+            WHERE triggered_by = 'manual'
+            ORDER BY generated_at DESC
+            LIMIT 1
+        """)
+    )
+    last_manual = result.fetchone()
+
+    if last_manual:
+        elapsed = datetime.now(timezone.utc) - last_manual.generated_at.replace(tzinfo=timezone.utc)
+        if elapsed.total_seconds() < MANUAL_RATE_LIMIT_MINUTES * 60:
+            remaining = MANUAL_RATE_LIMIT_MINUTES - (elapsed.total_seconds() / 60)
+            raise HTTPException(
+                status_code=429,
+                detail=f"Rate limited. Please wait {remaining:.0f} more minutes."
+            )
+
+    # Run generation
+    from jobs.ai_insights import generate_insight
+
+    result = await generate_insight(db, triggered_by="manual")
+
+    if result.get("success"):
+        return {
+            "success": True,
+            "content": result["content"],
+            "input_tokens": result["input_tokens"],
+            "output_tokens": result["output_tokens"],
+            "model": result["model"],
+        }
+    else:
+        raise HTTPException(
+            status_code=400,
+            detail=result.get("error", "Failed to generate insight")
+        )

backend/api/backup.py

@@ -0,0 +1,303 @@
+"""
+Backup and Restore API Endpoints
+"""
+import logging
+from fastapi import APIRouter, Depends, HTTPException, UploadFile, File
+from fastapi.responses import FileResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+from pathlib import Path
+
+from database import get_db
+from auth import get_current_user
+from services.backup_service import BackupService
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+# ============================================
+# HELPER FUNCTIONS
+# ============================================
+# Note: All backup endpoints require authentication but not admin role,
+# consistent with other sensitive operations in the application
+
+
+# ============================================
+# API ENDPOINTS
+# ============================================
+
+@router.get("/settings")
+async def get_backup_settings(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get backup configuration settings
+
+    Returns backup frequency, retention count, destination, and last backup status
+    """
+    try:
+        service = BackupService(db)
+        await service.ensure_backup_table_exists()
+        settings = await service.get_backup_settings()
+        return settings
+    except Exception as e:
+        logger.error(f"Failed to get backup settings: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.patch("/settings")
+async def update_backup_settings(
+    updates: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Update backup configuration settings
+
+    Body: {
+        frequency?: 'manual' | 'daily' | 'weekly' | 'monthly',
+        retention_count?: number,
+        destination?: 'local',
+        time?: 'HH:MM'
+    }
+    """
+    try:
+        service = BackupService(db)
+        success = await service.update_backup_settings(updates)
+
+        if success:
+            return {"message": "Settings updated successfully"}
+        else:
+            raise HTTPException(status_code=500, detail="Failed to update settings")
+
+    except Exception as e:
+        logger.error(f"Failed to update backup settings: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/create")
+async def create_backup(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger manual backup creation
+
+    Creates a ZIP file containing database dump, JSON export, and all data files.
+    Returns backup ID and status.
+    """
+    try:
+        service = BackupService(db)
+        await service.ensure_backup_table_exists()
+
+        username = current_user.get('username', 'unknown')
+        success, message, backup_id = await service.create_backup(
+            backup_type='manual',
+            created_by=username
+        )
+
+        if success:
+            return {
+                "success": True,
+                "message": message,
+                "backup_id": backup_id
+            }
+        else:
+            raise HTTPException(status_code=500, detail=message)
+
+    except Exception as e:
+        logger.error(f"Backup creation failed: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/history")
+async def get_backup_history(
+    limit: int = 50,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    List all backups, newest first
+
+    Query params:
+    - limit: Maximum number of backups to return (default 50)
+    """
+    try:
+        service = BackupService(db)
+        await service.ensure_backup_table_exists()
+        backups = await service.list_backups(limit=limit)
+        return backups
+    except Exception as e:
+        logger.error(f"Failed to get backup history: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/{backup_id}")
+async def get_backup(
+    backup_id: int,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get a specific backup by ID"""
+    try:
+        service = BackupService(db)
+        backup = await service.get_backup(backup_id)
+
+        if not backup:
+            raise HTTPException(status_code=404, detail="Backup not found")
+
+        return backup
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to get backup: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/{backup_id}/download")
+async def download_backup(
+    backup_id: int,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Download a backup file
+
+    Returns the backup ZIP file for download
+    """
+    try:
+        service = BackupService(db)
+        backup = await service.get_backup(backup_id)
+
+        if not backup:
+            raise HTTPException(status_code=404, detail="Backup not found")
+
+        if backup['status'] != 'success':
+            raise HTTPException(
+                status_code=400,
+                detail="Cannot download backup that did not complete successfully"
+            )
+
+        file_path = Path(backup['file_path'])
+        if not file_path.exists():
+            raise HTTPException(status_code=404, detail="Backup file not found on disk")
+
+        return FileResponse(
+            path=str(file_path),
+            filename=backup['filename'],
+            media_type='application/zip'
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to download backup: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/{backup_id}/restore")
+async def restore_backup(
+    backup_id: int,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Restore from an existing backup
+
+    WARNING: This will overwrite the current database with the backup data.
+    All current data will be replaced.
+    """
+    try:
+        service = BackupService(db)
+        success, message = await service.restore_from_backup(backup_id)
+
+        if success:
+            return {"success": True, "message": message}
+        else:
+            raise HTTPException(status_code=500, detail=message)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Restore failed: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/upload-restore")
+async def upload_and_restore(
+    file: UploadFile = File(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Upload and restore from a backup file
+
+    Upload a backup ZIP file and restore database and files from it.
+
+    WARNING: This will overwrite the current database with the backup data.
+    """
+    try:
+        # Validate file type
+        if not file.filename or not file.filename.endswith('.zip'):
+            raise HTTPException(
+                status_code=400,
+                detail="Only ZIP files are accepted"
+            )
+
+        # Read file content
+        file_content = await file.read()
+
+        # Validate file size (max 5GB)
+        max_size = 5 * 1024 * 1024 * 1024  # 5GB
+        if len(file_content) > max_size:
+            raise HTTPException(
+                status_code=400,
+                detail=f"File too large (max {max_size / 1024 / 1024 / 1024}GB)"
+            )
+
+        # Restore from uploaded file
+        service = BackupService(db)
+        success, message = await service.restore_from_upload(
+            file_content=file_content,
+            filename=file.filename
+        )
+
+        if success:
+            return {"success": True, "message": message}
+        else:
+            raise HTTPException(status_code=500, detail=message)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Upload restore failed: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/{backup_id}")
+async def delete_backup(
+    backup_id: int,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Delete a backup file and record
+
+    Removes the backup file from disk and deletes the database record.
+    """
+    try:
+        service = BackupService(db)
+        success, message = await service.delete_backup(backup_id)
+
+        if success:
+            return {"success": True, "message": message}
+        else:
+            raise HTTPException(status_code=500, detail=message)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to delete backup: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

backend/api/bookability.py

@@ -0,0 +1,598 @@
+"""
+Bookability API endpoints
+Rate availability matrix and competitor rate comparison
+"""
+from typing import Optional, List, Dict, Any
+from datetime import date, datetime, timedelta
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+import logging
+import json
+
+from database import get_db, SyncSessionLocal
+from auth import get_current_user
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# ============================================
+# RESPONSE MODELS
+# ============================================
+
+class CategoryInfo(BaseModel):
+    category_id: str
+    category_name: str
+    room_count: int
+
+
+class TariffInfo(BaseModel):
+    name: str
+    description: Optional[str] = None
+    rate: Optional[float] = None
+    average_nightly: Optional[float] = None
+    available: bool
+    message: str
+    sort_order: int = 999
+    min_stay: Optional[int] = None
+    available_for_min_stay: Optional[bool] = None  # True if available when queried with min_stay nights
+
+
+class OccupancyInfo(BaseModel):
+    occupied: int = 0
+    available: int = 0
+    maintenance: int = 0
+
+
+class DateRateInfo(BaseModel):
+    rate_gross: Optional[float] = None
+    rate_net: Optional[float] = None
+    tariffs: List[TariffInfo]
+    tariff_count: int
+    occupancy: Optional[OccupancyInfo] = None
+
+
+class RateMatrixResponse(BaseModel):
+    categories: List[CategoryInfo]
+    dates: List[str]
+    matrix: Dict[str, Dict[str, DateRateInfo]]
+
+
+# ============================================
+# RATE MATRIX ENDPOINT
+# ============================================
+
+@router.get("/rate-matrix", response_model=RateMatrixResponse)
+async def get_rate_matrix(
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+    category_id: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get rate availability matrix for all tariffs across dates and categories.
+
+    Returns a matrix showing all available tariff options for each room category
+    and date combination, including availability status and rates.
+
+    Args:
+        from_date: Start date (YYYY-MM-DD), defaults to today
+        to_date: End date (YYYY-MM-DD), defaults to today + 30 days
+        category_id: Optional filter to specific category
+
+    Returns:
+        RateMatrixResponse with categories, dates, and the matrix data
+    """
+    # Default date range
+    today = date.today()
+    start = date.fromisoformat(from_date) if from_date else today
+    end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
+
+    # Validate date range
+    if end < start:
+        raise HTTPException(status_code=400, detail="to_date must be after from_date")
+    if (end - start).days > 366:
+        raise HTTPException(status_code=400, detail="Date range cannot exceed 366 days")
+
+    # Fetch categories
+    cat_query = """
+        SELECT site_id, site_name, room_count
+        FROM newbook_room_categories
+        WHERE is_included = true
+    """
+    params: Dict[str, Any] = {}
+
+    if category_id:
+        cat_query += " AND site_id = :category_id"
+        params["category_id"] = category_id
+
+    cat_query += " ORDER BY display_order, site_name"
+
+    cat_result = await db.execute(text(cat_query), params)
+    categories = [
+        CategoryInfo(
+            category_id=row.site_id,
+            category_name=row.site_name,
+            room_count=row.room_count or 0
+        )
+        for row in cat_result.fetchall()
+    ]
+
+    if not categories:
+        return RateMatrixResponse(categories=[], dates=[], matrix={})
+
+    # Build date list
+    dates = []
+    current = start
+    while current <= end:
+        dates.append(current.isoformat())
+        current += timedelta(days=1)
+
+    # Fetch rates with tariffs_data (get latest version per category/date)
+    rates_query = """
+        SELECT DISTINCT ON (category_id, rate_date)
+            category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from
+        FROM newbook_current_rates
+        WHERE rate_date >= :from_date AND rate_date <= :to_date
+    """
+    rates_params: Dict[str, Any] = {"from_date": start, "to_date": end}
+
+    if category_id:
+        rates_query += " AND category_id = :category_id"
+        rates_params["category_id"] = category_id
+
+    rates_query += " ORDER BY category_id, rate_date, valid_from DESC"
+
+    rates_result = await db.execute(text(rates_query), rates_params)
+    rates_rows = rates_result.fetchall()
+
+    # Fetch occupancy data from newbook_occupancy_report_data
+    occupancy_query = """
+        SELECT category_id, date, occupied, available, maintenance
+        FROM newbook_occupancy_report_data
+        WHERE date >= :from_date AND date <= :to_date
+    """
+    occupancy_params: Dict[str, Any] = {"from_date": start, "to_date": end}
+
+    if category_id:
+        occupancy_query += " AND category_id = :category_id"
+        occupancy_params["category_id"] = category_id
+
+    occupancy_result = await db.execute(text(occupancy_query), occupancy_params)
+    occupancy_rows = occupancy_result.fetchall()
+
+    # Build occupancy lookup: category_id -> date -> OccupancyInfo
+    occupancy_map: Dict[str, Dict[str, OccupancyInfo]] = {}
+    for row in occupancy_rows:
+        cat_id = row.category_id
+        occ_date = row.date.isoformat()
+        if cat_id not in occupancy_map:
+            occupancy_map[cat_id] = {}
+        occupancy_map[cat_id][occ_date] = OccupancyInfo(
+            occupied=row.occupied or 0,
+            available=row.available or 0,
+            maintenance=row.maintenance or 0
+        )
+
+    # Build matrix
+    matrix: Dict[str, Dict[str, DateRateInfo]] = {}
+
+    # Initialize matrix with empty data for all categories and dates
+    for cat in categories:
+        matrix[cat.category_id] = {}
+        for date_str in dates:
+            # Get occupancy for this category/date if available
+            occ = occupancy_map.get(cat.category_id, {}).get(date_str)
+            matrix[cat.category_id][date_str] = DateRateInfo(
+                rate_gross=None,
+                rate_net=None,
+                tariffs=[],
+                tariff_count=0,
+                occupancy=occ
+            )
+
+    # Populate matrix with actual data
+    for row in rates_rows:
+        cat_id = row.category_id
+        rate_date = row.rate_date.isoformat()
+
+        if cat_id not in matrix or rate_date not in matrix[cat_id]:
+            continue
+
+        # Parse tariffs_data
+        tariffs_data = row.tariffs_data or {}
+        if isinstance(tariffs_data, str):
+            try:
+                tariffs_data = json.loads(tariffs_data)
+            except json.JSONDecodeError:
+                tariffs_data = {}
+
+        # Build tariff list
+        tariffs_list = []
+        raw_tariffs = tariffs_data.get('tariffs', [])
+
+        for idx, tariff in enumerate(raw_tariffs):
+            tariffs_list.append(TariffInfo(
+                name=tariff.get('name', 'Unknown'),
+                description=tariff.get('description'),
+                rate=tariff.get('rate'),
+                average_nightly=tariff.get('average_nightly'),
+                available=tariff.get('success', False),
+                message=tariff.get('message', ''),
+                sort_order=tariff.get('sort_order', idx),
+                min_stay=tariff.get('min_stay'),
+                available_for_min_stay=tariff.get('available_for_min_stay')
+            ))
+
+        # Preserve existing occupancy data
+        existing_occ = matrix[cat_id][rate_date].occupancy
+
+        matrix[cat_id][rate_date] = DateRateInfo(
+            rate_gross=float(row.rate_gross) if row.rate_gross else None,
+            rate_net=float(row.rate_net) if row.rate_net else None,
+            tariffs=tariffs_list,
+            tariff_count=tariffs_data.get('tariff_count', len(tariffs_list)),
+            occupancy=existing_occ
+        )
+
+    return RateMatrixResponse(
+        categories=categories,
+        dates=dates,
+        matrix=matrix
+    )
+
+
+# ============================================
+# RATE MATRIX SUMMARY (lightweight endpoint)
+# ============================================
+
+@router.get("/rate-matrix/summary")
+async def get_rate_matrix_summary(
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get a summary of rate availability issues.
+
+    Returns counts of unavailable tariffs by category and date for quick
+    identification of potential bookability problems.
+    """
+    today = date.today()
+    start = date.fromisoformat(from_date) if from_date else today
+    end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
+
+    # Query rates with issues (get latest version per category/date)
+    result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (category_id, rate_date)
+                category_id,
+                rate_date,
+                tariffs_data
+            FROM newbook_current_rates
+            WHERE rate_date >= :from_date AND rate_date <= :to_date
+            AND tariffs_data IS NOT NULL
+            ORDER BY category_id, rate_date, valid_from DESC
+        """),
+        {"from_date": start, "to_date": end}
+    )
+
+    issues = []
+    for row in result.fetchall():
+        tariffs_data = row.tariffs_data or {}
+        if isinstance(tariffs_data, str):
+            try:
+                tariffs_data = json.loads(tariffs_data)
+            except json.JSONDecodeError:
+                continue
+
+        tariffs = tariffs_data.get('tariffs', [])
+        unavailable = [t for t in tariffs if not t.get('success', False)]
+
+        if unavailable:
+            issues.append({
+                "category_id": row.category_id,
+                "date": row.rate_date.isoformat(),
+                "unavailable_count": len(unavailable),
+                "unavailable_tariffs": [t.get('name') for t in unavailable],
+                "messages": [t.get('message') for t in unavailable if t.get('message')]
+            })
+
+    return {
+        "from_date": start.isoformat(),
+        "to_date": end.isoformat(),
+        "total_issues": len(issues),
+        "issues": issues
+    }
+
+
+# ============================================
+# RATE HISTORY ENDPOINT
+# ============================================
+
+@router.get("/rate-history/{category_id}/{rate_date}")
+async def get_rate_history(
+    category_id: str,
+    rate_date: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get rate change history for a specific category and date.
+
+    Returns all rate snapshots showing how rates evolved over time.
+    Useful for understanding when rates changed and by how much.
+    """
+    try:
+        target_date = date.fromisoformat(rate_date)
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    result = await db.execute(
+        text("""
+            SELECT
+                id,
+                rate_gross,
+                rate_net,
+                tariffs_data,
+                valid_from,
+                last_verified_at
+            FROM newbook_current_rates
+            WHERE category_id = :category_id AND rate_date = :rate_date
+            ORDER BY valid_from DESC
+        """),
+        {"category_id": category_id, "rate_date": target_date}
+    )
+
+    history = []
+    for row in result.fetchall():
+        tariffs_data = row.tariffs_data or {}
+        if isinstance(tariffs_data, str):
+            try:
+                tariffs_data = json.loads(tariffs_data)
+            except json.JSONDecodeError:
+                tariffs_data = {}
+
+        tariffs = tariffs_data.get('tariffs', [])
+
+        history.append({
+            "id": row.id,
+            "rate_gross": float(row.rate_gross) if row.rate_gross else None,
+            "rate_net": float(row.rate_net) if row.rate_net else None,
+            "valid_from": row.valid_from.isoformat() if row.valid_from else None,
+            "last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None,
+            "tariff_count": len(tariffs),
+            "tariffs_available": sum(1 for t in tariffs if t.get('success', False)),
+            "tariffs_unavailable": sum(1 for t in tariffs if not t.get('success', False)),
+            "tariffs": [
+                {
+                    "name": t.get('name'),
+                    "rate": t.get('rate'),
+                    "available": t.get('success', False),
+                    "message": t.get('message', ''),
+                    "min_stay": t.get('min_stay')
+                }
+                for t in tariffs
+            ]
+        })
+
+    return {
+        "category_id": category_id,
+        "rate_date": rate_date,
+        "version_count": len(history),
+        "history": history
+    }
+
+
+# ============================================
+# RATE CHANGES SUMMARY
+# ============================================
+
+@router.get("/rate-changes")
+async def get_rate_changes(
+    days: int = 7,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get summary of rate changes in the last N days.
+
+    Shows which rates changed and when, useful for tracking pricing strategy changes.
+    """
+    cutoff = datetime.now() - timedelta(days=days)
+
+    # Find dates with multiple versions (indicating changes)
+    result = await db.execute(
+        text("""
+            SELECT
+                category_id,
+                rate_date,
+                COUNT(*) as version_count,
+                MIN(valid_from) as first_version,
+                MAX(valid_from) as latest_version
+            FROM newbook_current_rates
+            WHERE valid_from >= :cutoff
+            GROUP BY category_id, rate_date
+            HAVING COUNT(*) > 1
+            ORDER BY MAX(valid_from) DESC
+            LIMIT 100
+        """),
+        {"cutoff": cutoff}
+    )
+
+    changes = []
+    for row in result.fetchall():
+        changes.append({
+            "category_id": row.category_id,
+            "rate_date": row.rate_date.isoformat(),
+            "version_count": row.version_count,
+            "first_version": row.first_version.isoformat() if row.first_version else None,
+            "latest_version": row.latest_version.isoformat() if row.latest_version else None
+        })
+
+    return {
+        "days": days,
+        "total_changes": len(changes),
+        "changes": changes
+    }
+
+
+# ============================================
+# FETCH RATES TRIGGER (manual refresh)
+# ============================================
+
+@router.post("/refresh-rates")
+async def refresh_rates(
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+    category_id: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger a manual refresh of current rates from Newbook.
+
+    This runs the rate fetch job for the specified date range.
+    Note: This can be slow as it respects Newbook API rate limits.
+    """
+    from jobs.fetch_current_rates import run_fetch_current_rates
+
+    # For now, just run the standard fetch
+    # TODO: Add support for custom date range and category filter
+    try:
+        await run_fetch_current_rates()
+        return {"status": "success", "message": "Rates refresh completed"}
+    except Exception as e:
+        logger.error(f"Rates refresh failed: {e}")
+        raise HTTPException(status_code=500, detail=f"Rates refresh failed: {str(e)}")
+
+
+# ============================================
+# SINGLE-DATE RATE REFRESH
+# ============================================
+
+def _refresh_date_sync(rate_date: date):
+    """
+    Fetch rates for a single date from Newbook and save to DB.
+    Runs synchronously in a background task.
+    """
+    import asyncio
+    from decimal import Decimal
+    from services.newbook_rates_client import NewbookRatesClient
+    from jobs.fetch_current_rates import save_rate_snapshot
+
+    db = SyncSessionLocal()
+    try:
+        # Get config
+        config_result = db.execute(
+            text("""
+                SELECT config_key, config_value FROM system_config
+                WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')
+            """)
+        )
+        config = {row.config_key: row.config_value for row in config_result.fetchall()}
+
+        if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
+            logger.error("Newbook credentials not configured for single-date refresh")
+            return
+
+        vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20'))
+
+        # Get included categories
+        cat_result = db.execute(
+            text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
+        )
+        included_categories = set(row.site_id for row in cat_result.fetchall())
+
+        client = NewbookRatesClient(
+            api_key=config['newbook_api_key'],
+            username=config['newbook_username'],
+            password=config['newbook_password'],
+            region=config['newbook_region'],
+            vat_rate=vat_rate
+        )
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            async def _fetch():
+                async with client:
+                    # Single-night query for this date (all categories)
+                    category_rates = await client._fetch_all_categories_batch(
+                        rate_date, guests_adults=2, guests_children=0
+                    )
+
+                    # Check for min_stay tariffs needing multi-night verification
+                    dates_by_nights: Dict[int, list] = {}
+                    for cat_id, rates in category_rates.items():
+                        if cat_id not in included_categories:
+                            continue
+                        for rate in rates:
+                            for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
+                                min_stay = tariff.get('min_stay')
+                                if min_stay and min_stay > 1 and not tariff.get('success', False):
+                                    if min_stay not in dates_by_nights:
+                                        dates_by_nights[min_stay] = []
+                                    if rate_date not in dates_by_nights[min_stay]:
+                                        dates_by_nights[min_stay].append(rate_date)
+
+                    # Run multi-night verification if needed
+                    if dates_by_nights:
+                        multi_results = await client.get_multi_night_availability(dates_by_nights)
+                        for cat_id, rates in category_rates.items():
+                            if cat_id not in included_categories:
+                                continue
+                            for rate in rates:
+                                if rate_date in multi_results:
+                                    cat_avail = multi_results[rate_date].get(cat_id, {})
+                                    for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
+                                        if tariff.get('min_stay') and tariff['min_stay'] > 1:
+                                            tariff['available_for_min_stay'] = cat_avail.get(tariff.get('name', ''), False)
+
+                    # Save snapshots
+                    inserted = 0
+                    for cat_id, rates in category_rates.items():
+                        if cat_id not in included_categories:
+                            continue
+                        for rate in rates:
+                            result = save_rate_snapshot(db, cat_id, rate['date'], rate)
+                            if result == 'inserted':
+                                inserted += 1
+
+                    return inserted
+
+            inserted = loop.run_until_complete(_fetch())
+            db.commit()
+            logger.info(f"Single-date refresh for {rate_date}: {inserted} new snapshots")
+
+        finally:
+            loop.close()
+
+    except Exception as e:
+        logger.error(f"Single-date refresh failed for {rate_date}: {e}", exc_info=True)
+        db.rollback()
+    finally:
+        db.close()
+
+
+@router.post("/refresh-date/{rate_date}")
+async def refresh_single_date(
+    rate_date: str,
+    background_tasks: BackgroundTasks,
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger a refresh of rates for a single date from Newbook.
+    Runs in background - returns immediately.
+    """
+    try:
+        target_date = date.fromisoformat(rate_date)
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    background_tasks.add_task(_refresh_date_sync, target_date)
+    return {"status": "queued", "date": rate_date, "message": f"Refreshing rates for {rate_date}"}

backend/api/budget.py

@@ -0,0 +1,559 @@
+"""
+Budget API endpoints
+"""
+import io
+import re
+import logging
+from datetime import date, datetime
+from typing import Optional, List
+
+import pandas as pd
+from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File
+from fastapi.responses import StreamingResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+
+from database import get_db
+from auth import get_current_user
+
+logger = logging.getLogger(__name__)
+
+# Mapping from spreadsheet row labels to budget_type values
+BUDGET_TYPE_MAPPING = {
+    'accom': 'net_accom',
+    'accommodation': 'net_accom',
+    'acc': 'net_accom',
+    'dry': 'net_dry',
+    'food': 'net_dry',
+    'wet': 'net_wet',
+    'beverage': 'net_wet',
+    'beverages': 'net_wet',
+}
+
+router = APIRouter()
+
+
+class MonthlyBudgetCreate(BaseModel):
+    year: int
+    month: int
+    budget_type: str
+    budget_value: float
+    notes: Optional[str] = None
+
+
+class MonthlyBudgetResponse(BaseModel):
+    id: int
+    year: int
+    month: int
+    budget_type: str
+    budget_value: float
+    notes: Optional[str]
+
+
+@router.get("/monthly")
+async def get_monthly_budgets(
+    year: int = Query(..., description="Year to get budgets for"),
+    budget_type: Optional[str] = Query(None, description="Filter by budget type"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get monthly budgets for a year.
+    """
+    query = """
+        SELECT id, year, month, budget_type, budget_value, notes, created_at, updated_at
+        FROM monthly_budgets
+        WHERE year = :year
+    """
+    params = {"year": year}
+
+    if budget_type:
+        query += " AND budget_type = :budget_type"
+        params["budget_type"] = budget_type
+
+    query += " ORDER BY month, budget_type"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "id": row.id,
+            "year": row.year,
+            "month": row.month,
+            "budget_type": row.budget_type,
+            "budget_value": float(row.budget_value),
+            "notes": row.notes,
+            "created_at": row.created_at,
+            "updated_at": row.updated_at
+        }
+        for row in rows
+    ]
+
+
+@router.post("/monthly")
+async def create_or_update_monthly_budget(
+    budget: MonthlyBudgetCreate,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Create or update a monthly budget.
+    """
+    query = """
+        INSERT INTO monthly_budgets (year, month, budget_type, budget_value, notes, updated_at)
+        VALUES (:year, :month, :budget_type, :budget_value, :notes, NOW())
+        ON CONFLICT (year, month, budget_type)
+        DO UPDATE SET budget_value = :budget_value, notes = :notes, updated_at = NOW()
+        RETURNING id
+    """
+
+    result = await db.execute(text(query), {
+        "year": budget.year,
+        "month": budget.month,
+        "budget_type": budget.budget_type,
+        "budget_value": budget.budget_value,
+        "notes": budget.notes
+    })
+    await db.commit()
+
+    row = result.fetchone()
+    return {"id": row.id, "status": "saved", **budget.model_dump()}
+
+
+@router.get("/daily")
+async def get_daily_budgets(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    budget_type: Optional[str] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get daily distributed budgets for a date range.
+    """
+    query = """
+        SELECT
+            date,
+            budget_type,
+            budget_value,
+            distribution_method,
+            prior_year_pct
+        FROM daily_budgets
+        WHERE date BETWEEN :from_date AND :to_date
+    """
+    params = {"from_date": from_date, "to_date": to_date}
+
+    if budget_type:
+        query += " AND budget_type = :budget_type"
+        params["budget_type"] = budget_type
+
+    query += " ORDER BY date, budget_type"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.date,
+            "budget_type": row.budget_type,
+            "budget_value": float(row.budget_value),
+            "distribution_method": row.distribution_method,
+            "prior_year_pct": float(row.prior_year_pct) if row.prior_year_pct else None
+        }
+        for row in rows
+    ]
+
+
+@router.post("/distribute")
+async def distribute_monthly_budget(
+    year: int = Query(...),
+    month: int = Query(...),
+    budget_type: Optional[str] = Query(None, description="Budget type to distribute, or all if not specified"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Distribute monthly budget to daily values using prior year patterns.
+    """
+    from services.forecasting.budget_service import distribute_budget
+
+    result = await distribute_budget(db, year, month, budget_type)
+
+    return {
+        "status": "distributed",
+        "year": year,
+        "month": month,
+        "budget_type": budget_type or "all",
+        "days_distributed": result.get("days_distributed", 0)
+    }
+
+
+@router.get("/variance")
+async def get_budget_variance(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get forecast vs budget vs actual variance for date range.
+    """
+    query = """
+        SELECT
+            db.date,
+            db.budget_type,
+            db.budget_value,
+            f.predicted_value as forecast_value,
+            dm.actual_value,
+            (f.predicted_value - db.budget_value) as forecast_vs_budget,
+            CASE WHEN db.budget_value != 0 THEN
+                ROUND(((f.predicted_value - db.budget_value) / db.budget_value * 100)::numeric, 2)
+            END as forecast_vs_budget_pct,
+            CASE WHEN dm.actual_value IS NOT NULL THEN
+                (dm.actual_value - db.budget_value)
+            END as actual_vs_budget,
+            CASE WHEN dm.actual_value IS NOT NULL AND db.budget_value != 0 THEN
+                ROUND(((dm.actual_value - db.budget_value) / db.budget_value * 100)::numeric, 2)
+            END as actual_vs_budget_pct
+        FROM daily_budgets db
+        LEFT JOIN forecasts f ON db.date = f.forecast_date
+            AND db.budget_type = f.forecast_type
+            AND f.model_type = 'prophet'
+        LEFT JOIN daily_metrics dm ON db.date = dm.date
+            AND db.budget_type = dm.metric_code
+        WHERE db.date BETWEEN :from_date AND :to_date
+        ORDER BY db.date, db.budget_type
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.date,
+            "budget_type": row.budget_type,
+            "budget": float(row.budget_value) if row.budget_value else None,
+            "forecast": float(row.forecast_value) if row.forecast_value else None,
+            "actual": float(row.actual_value) if row.actual_value else None,
+            "forecast_vs_budget": float(row.forecast_vs_budget) if row.forecast_vs_budget else None,
+            "forecast_vs_budget_pct": float(row.forecast_vs_budget_pct) if row.forecast_vs_budget_pct else None,
+            "actual_vs_budget": float(row.actual_vs_budget) if row.actual_vs_budget else None,
+            "actual_vs_budget_pct": float(row.actual_vs_budget_pct) if row.actual_vs_budget_pct else None
+        }
+        for row in rows
+    ]
+
+
+def parse_month_header(header) -> Optional[tuple]:
+    """
+    Parse month header in various formats to (year, month).
+
+    Supported formats:
+    - datetime/Timestamp objects (from Excel date cells)
+    - mm/yy (01/25, 12/26)
+    - mm-yy (01-25, 12-26)
+    - mmm/yy (Jan/25, Dec-26)
+    - mmm yy (Jan 25, Dec 26)
+    - yyyy-mm (2025-01)
+    - dd/mm/yyyy or mm/dd/yyyy (will use first of month)
+    - Full month names (January 2025)
+
+    Returns None if cannot parse.
+    """
+    if header is None:
+        return None
+
+    # Handle pandas NaT or NaN
+    if pd.isna(header):
+        return None
+
+    # Handle datetime objects (from Excel date columns)
+    if isinstance(header, (datetime, date)):
+        return (header.year, header.month)
+
+    # Handle pandas Timestamp
+    if hasattr(header, 'year') and hasattr(header, 'month'):
+        try:
+            return (int(header.year), int(header.month))
+        except (ValueError, TypeError):
+            pass
+
+    # Convert to string for text parsing
+    if not isinstance(header, str):
+        header = str(header)
+
+    header = header.strip()
+    if not header:
+        return None
+
+    # Try mm/yy or mm-yy format
+    match = re.match(r'^(\d{1,2})[/\-](\d{2,4})$', header)
+    if match:
+        month = int(match.group(1))
+        year = int(match.group(2))
+        if year < 100:
+            year = 2000 + year if year < 50 else 1900 + year
+        if 1 <= month <= 12:
+            return (year, month)
+
+    # Try yyyy-mm format
+    match = re.match(r'^(\d{4})[/\-](\d{1,2})$', header)
+    if match:
+        year = int(match.group(1))
+        month = int(match.group(2))
+        if 1 <= month <= 12:
+            return (year, month)
+
+    # Try dd/mm/yyyy or yyyy-mm-dd format (use year/month, ignore day)
+    match = re.match(r'^(\d{1,2})[/\-](\d{1,2})[/\-](\d{4})$', header)
+    if match:
+        # Assume dd/mm/yyyy
+        day = int(match.group(1))
+        month = int(match.group(2))
+        year = int(match.group(3))
+        if 1 <= month <= 12:
+            return (year, month)
+
+    match = re.match(r'^(\d{4})[/\-](\d{1,2})[/\-](\d{1,2})$', header)
+    if match:
+        year = int(match.group(1))
+        month = int(match.group(2))
+        if 1 <= month <= 12:
+            return (year, month)
+
+    # Try month name formats (Jan/25, Jan-25, Jan 25, Jan25)
+    month_names = {
+        'jan': 1, 'feb': 2, 'mar': 3, 'apr': 4, 'may': 5, 'jun': 6,
+        'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12,
+        'january': 1, 'february': 2, 'march': 3, 'april': 4, 'june': 6,
+        'july': 7, 'august': 8, 'september': 9, 'october': 10, 'november': 11, 'december': 12
+    }
+    match = re.match(r'^([a-zA-Z]+)[/\-\s]?(\d{2,4})$', header, re.IGNORECASE)
+    if match:
+        month_str = match.group(1).lower()
+        year = int(match.group(2))
+        if year < 100:
+            year = 2000 + year if year < 50 else 1900 + year
+        if month_str in month_names:
+            return (year, month_names[month_str])
+
+    # Try "2025 January" or "2025-January" format
+    match = re.match(r'^(\d{4})[/\-\s]?([a-zA-Z]+)$', header, re.IGNORECASE)
+    if match:
+        year = int(match.group(1))
+        month_str = match.group(2).lower()
+        if month_str in month_names:
+            return (year, month_names[month_str])
+
+    return None
+
+
+def clean_numeric_value(value) -> Optional[float]:
+    """
+    Clean a value that might contain currency symbols, commas, etc.
+    Returns None if the value cannot be converted to a number.
+    """
+    if value is None or (isinstance(value, float) and pd.isna(value)):
+        return None
+
+    if isinstance(value, (int, float)):
+        return float(value)
+
+    if isinstance(value, str):
+        # Remove currency symbols, commas, spaces
+        cleaned = re.sub(r'[£$€,\s]', '', value.strip())
+        if cleaned == '' or cleaned == '-':
+            return None
+        try:
+            return float(cleaned)
+        except ValueError:
+            return None
+
+    return None
+
+
+@router.post("/upload")
+async def upload_budget_spreadsheet(
+    file: UploadFile = File(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Upload budget spreadsheet (CSV/Excel) in format:
+
+    month | 01/25  | 02/25  | 03/25  | ...
+    accom | 150000 | 145000 | 160000 | ...
+    dry   | 45000  | 42000  | 48000  | ...
+    wet   | 35000  | 32000  | 38000  | ...
+
+    Returns summary of records created/updated.
+    """
+    # Validate file type
+    filename = file.filename.lower()
+    if not (filename.endswith('.csv') or filename.endswith('.xlsx') or filename.endswith('.xls')):
+        raise HTTPException(
+            status_code=400,
+            detail="Invalid file type. Please upload a CSV or Excel file (.csv, .xlsx, .xls)"
+        )
+
+    # Read file content
+    content = await file.read()
+
+    try:
+        # Parse file based on type
+        if filename.endswith('.csv'):
+            df = pd.read_csv(io.BytesIO(content), header=None)
+        else:
+            df = pd.read_excel(io.BytesIO(content), header=None)
+    except Exception as e:
+        logger.error(f"Failed to parse budget file: {e}")
+        raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
+
+    if df.empty:
+        raise HTTPException(status_code=400, detail="File is empty")
+
+    # Parse the spreadsheet structure
+    # First row should contain month headers (skip first column which is the label column)
+    # Subsequent rows contain budget type label and values
+
+    records_created = 0
+    records_updated = 0
+    errors = []
+
+    # Get month headers from first row (skip first column)
+    month_headers = df.iloc[0, 1:].tolist()
+    parsed_months = []
+
+    logger.info(f"Found {len(month_headers)} column headers")
+    for idx, header in enumerate(month_headers):
+        logger.debug(f"Header {idx}: {header} (type: {type(header).__name__})")
+        parsed = parse_month_header(header)  # Pass raw value, parser handles types
+        if parsed:
+            parsed_months.append((idx + 1, parsed))  # Store column index and (year, month)
+            logger.debug(f"  -> Parsed as {parsed[0]}-{parsed[1]:02d}")
+        else:
+            if header is not None and not pd.isna(header) and str(header).strip():
+                errors.append(f"Could not parse month header: '{header}' (type: {type(header).__name__})")
+
+    if not parsed_months:
+        # Log what we received for debugging
+        sample_headers = month_headers[:5] if len(month_headers) > 5 else month_headers
+        logger.error(f"No valid month headers found. Sample headers: {sample_headers}")
+        raise HTTPException(
+            status_code=400,
+            detail=f"No valid month headers found. Got: {sample_headers}. Expected formats: mm/yy, Jan-25, 2025-01, or Excel dates"
+        )
+
+    # Process budget rows (skip first header row)
+    for row_idx in range(1, len(df)):
+        row = df.iloc[row_idx]
+        row_label = str(row.iloc[0]).lower().strip() if row.iloc[0] else ''
+
+        # Map row label to budget_type
+        budget_type = BUDGET_TYPE_MAPPING.get(row_label)
+        if not budget_type:
+            if row_label and row_label not in ['month', 'total', '']:
+                errors.append(f"Unknown budget type: '{row_label}'")
+            continue
+
+        # Process each month column
+        for col_idx, (year, month) in parsed_months:
+            value = clean_numeric_value(row.iloc[col_idx])
+            if value is None:
+                continue
+
+            # Upsert the budget value
+            try:
+                result = await db.execute(
+                    text("""
+                        INSERT INTO monthly_budgets (year, month, budget_type, budget_value, updated_at)
+                        VALUES (:year, :month, :budget_type, :budget_value, NOW())
+                        ON CONFLICT (year, month, budget_type)
+                        DO UPDATE SET budget_value = :budget_value, updated_at = NOW()
+                        RETURNING (xmax = 0) as inserted
+                    """),
+                    {
+                        "year": year,
+                        "month": month,
+                        "budget_type": budget_type,
+                        "budget_value": value
+                    }
+                )
+                row_result = result.fetchone()
+                if row_result and row_result.inserted:
+                    records_created += 1
+                else:
+                    records_updated += 1
+            except Exception as e:
+                errors.append(f"Failed to save {budget_type} for {month:02d}/{year}: {str(e)}")
+
+    await db.commit()
+
+    logger.info(f"Budget upload complete: {records_created} created, {records_updated} updated")
+
+    return {
+        "status": "success",
+        "filename": file.filename,
+        "records_created": records_created,
+        "records_updated": records_updated,
+        "total_records": records_created + records_updated,
+        "errors": errors if errors else None
+    }
+
+
+@router.get("/template")
+async def download_budget_template(
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Download empty budget template Excel file.
+    Pre-fills month headers for current and next year.
+    """
+    # Generate month headers for current year and next year
+    current_year = datetime.now().year
+    months = []
+
+    for year in [current_year, current_year + 1]:
+        for month in range(1, 13):
+            months.append(f"{month:02d}/{year % 100:02d}")
+
+    # Create DataFrame with template structure
+    data = {
+        'Type': ['accom', 'dry', 'wet']
+    }
+
+    # Add empty columns for each month
+    for month_header in months:
+        data[month_header] = ['', '', '']
+
+    df = pd.DataFrame(data)
+
+    # Write to Excel
+    output = io.BytesIO()
+    with pd.ExcelWriter(output, engine='openpyxl') as writer:
+        df.to_excel(writer, sheet_name='Budget', index=False)
+
+        # Auto-adjust column widths
+        worksheet = writer.sheets['Budget']
+        for column in worksheet.columns:
+            max_length = 0
+            column_letter = column[0].column_letter
+            for cell in column:
+                try:
+                    if len(str(cell.value)) > max_length:
+                        max_length = len(str(cell.value))
+                except:
+                    pass
+            worksheet.column_dimensions[column_letter].width = max(max_length + 2, 10)
+
+    output.seek(0)
+
+    return StreamingResponse(
+        output,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers={
+            "Content-Disposition": f"attachment; filename=budget_template_{current_year}.xlsx"
+        }
+    )

backend/api/competitor_rates.py

@@ -0,0 +1,940 @@
+"""
+Competitor Rates API endpoints
+Booking.com rate scraping, hotel management, and competitor comparison
+"""
+from typing import Optional, List, Dict, Any
+from datetime import date, datetime, timedelta
+from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+import logging
+
+from database import get_db, SyncSessionLocal
+from auth import get_current_user
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# ============================================
+# REQUEST/RESPONSE MODELS
+# ============================================
+
+class ScrapeRequest(BaseModel):
+    from_date: str
+    to_date: Optional[str] = None
+
+
+class LocationConfigRequest(BaseModel):
+    location_name: str
+    pages_to_scrape: int = 2
+    adults: int = 2
+
+
+class HotelTierUpdate(BaseModel):
+    tier: str  # 'own', 'competitor', 'market'
+    display_order: Optional[int] = None
+
+
+class HotelResponse(BaseModel):
+    id: int
+    booking_com_id: str
+    name: str
+    booking_com_url: Optional[str]
+    star_rating: Optional[float]
+    review_score: Optional[float]
+    review_count: Optional[int]
+    tier: str
+    display_order: int
+    notes: Optional[str]
+    first_seen_at: Optional[datetime]
+    last_seen_at: Optional[datetime]
+
+
+class RateResponse(BaseModel):
+    rate_date: str
+    hotel_id: int
+    hotel_name: str
+    tier: str
+    star_rating: Optional[float]
+    review_score: Optional[float]
+    availability_status: str
+    rate_gross: Optional[float]
+    room_type: Optional[str]
+    breakfast_included: Optional[bool]
+    free_cancellation: Optional[bool]
+    no_prepayment: Optional[bool]
+    rooms_left: Optional[int]
+    scraped_at: Optional[datetime]
+
+
+class ScraperStatusResponse(BaseModel):
+    enabled: bool
+    paused: bool
+    pause_until: Optional[str]
+    backend: str
+    location_configured: bool
+    location_name: Optional[str]
+    last_scrape: Optional[dict]
+
+
+# ============================================
+# SCRAPER STATUS & CONFIGURATION
+# ============================================
+
+@router.get("/status", response_model=ScraperStatusResponse)
+async def get_scraper_status(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get current scraper status and configuration."""
+    # Get config values
+    config_result = await db.execute(
+        text("""
+            SELECT config_key, config_value FROM system_config
+            WHERE config_key IN (
+                'booking_scraper_enabled',
+                'booking_scraper_paused',
+                'booking_scraper_pause_until',
+                'booking_scraper_backend'
+            )
+        """)
+    )
+    config = {row.config_key: row.config_value for row in config_result.fetchall()}
+
+    # Get location config
+    location_result = await db.execute(
+        text("SELECT location_name FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
+    )
+    location_row = location_result.fetchone()
+
+    # Get last scrape info
+    last_scrape_result = await db.execute(
+        text("""
+            SELECT batch_id, scrape_type, started_at, completed_at, status,
+                   hotels_found, rates_scraped, error_message
+            FROM booking_scrape_log
+            ORDER BY started_at DESC
+            LIMIT 1
+        """)
+    )
+    last_scrape_row = last_scrape_result.fetchone()
+    last_scrape = None
+    if last_scrape_row:
+        last_scrape = {
+            'batch_id': str(last_scrape_row.batch_id),
+            'scrape_type': last_scrape_row.scrape_type,
+            'started_at': last_scrape_row.started_at.isoformat() if last_scrape_row.started_at else None,
+            'completed_at': last_scrape_row.completed_at.isoformat() if last_scrape_row.completed_at else None,
+            'status': last_scrape_row.status,
+            'hotels_found': last_scrape_row.hotels_found,
+            'rates_scraped': last_scrape_row.rates_scraped,
+            'error_message': last_scrape_row.error_message,
+        }
+
+    return ScraperStatusResponse(
+        enabled=config.get('booking_scraper_enabled', 'false') == 'true',
+        paused=config.get('booking_scraper_paused', 'false') == 'true',
+        pause_until=config.get('booking_scraper_pause_until'),
+        backend=config.get('booking_scraper_backend', 'playwright_local'),
+        location_configured=location_row is not None,
+        location_name=location_row.location_name if location_row else None,
+        last_scrape=last_scrape
+    )
+
+
+@router.post("/config/location")
+async def set_location_config(
+    config: LocationConfigRequest,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Set the location to scrape for competitor rates."""
+    # Deactivate existing configs
+    await db.execute(
+        text("UPDATE booking_scrape_config SET is_active = FALSE")
+    )
+
+    # Insert new config
+    await db.execute(
+        text("""
+            INSERT INTO booking_scrape_config (location_name, pages_to_scrape, adults, is_active)
+            VALUES (:location, :pages, :adults, TRUE)
+        """),
+        {'location': config.location_name, 'pages': config.pages_to_scrape, 'adults': config.adults}
+    )
+    await db.commit()
+
+    return {"status": "success", "location": config.location_name}
+
+
+@router.post("/config/enable")
+async def enable_scraper(
+    enabled: bool = True,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Enable or disable the booking.com scraper."""
+    await db.execute(
+        text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_enabled'"),
+        {'val': 'true' if enabled else 'false'}
+    )
+    await db.commit()
+    return {"status": "success", "enabled": enabled}
+
+
+@router.post("/config/unpause")
+async def unpause_scraper(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Manually unpause the scraper (clears blocking pause)."""
+    await db.execute(
+        text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
+    )
+    await db.commit()
+    return {"status": "success", "message": "Scraper unpaused"}
+
+
+# ============================================
+# MANUAL SCRAPE TRIGGER
+# ============================================
+
+def run_scrape_sync(from_date: date, to_date: date):
+    """Run scrape in sync context for background task."""
+    import asyncio
+    from services.booking_scraper import run_manual_scrape, cleanup_stale_batches
+
+    db = SyncSessionLocal()
+    try:
+        # Clean up any stale batches before starting
+        cleanup_stale_batches(db, max_age_minutes=60)
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            result = loop.run_until_complete(run_manual_scrape(db, from_date, to_date))
+            logger.info(f"Background scrape completed: {result}")
+        finally:
+            loop.close()
+    except Exception as e:
+        logger.error(f"Background scrape failed: {e}", exc_info=True)
+    finally:
+        db.close()
+
+
+@router.post("/scrape")
+async def trigger_manual_scrape(
+    request: ScrapeRequest,
+    background_tasks: BackgroundTasks,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger a manual scrape for the specified date range.
+
+    Runs in background - check /status for progress.
+    """
+    try:
+        from_date = date.fromisoformat(request.from_date)
+        to_date = date.fromisoformat(request.to_date) if request.to_date else from_date
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=f"Invalid date format: {e}")
+
+    if to_date < from_date:
+        raise HTTPException(status_code=400, detail="to_date must be after from_date")
+
+    if (to_date - from_date).days > 30:
+        raise HTTPException(status_code=400, detail="Date range cannot exceed 30 days for manual scrape")
+
+    # Check if location is configured
+    location_result = await db.execute(
+        text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
+    )
+    if not location_result.fetchone():
+        raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
+
+    # Check if paused
+    paused_result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
+    )
+    paused_row = paused_result.fetchone()
+    if paused_row and paused_row.config_value == 'true':
+        raise HTTPException(status_code=400, detail="Scraper is currently paused. Use /unpause first or wait for cooldown.")
+
+    # Start background task
+    background_tasks.add_task(run_scrape_sync, from_date, to_date)
+
+    return {
+        "status": "started",
+        "from_date": from_date.isoformat(),
+        "to_date": to_date.isoformat(),
+        "message": "Scrape started in background. Check /status for progress."
+    }
+
+
+# ============================================
+# HOTELS MANAGEMENT
+# ============================================
+
+@router.get("/hotels", response_model=List[HotelResponse])
+async def list_hotels(
+    tier: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    List all discovered hotels.
+
+    Filter by tier: 'own', 'competitor', 'market', or None for all.
+    """
+    query = """
+        SELECT id, booking_com_id, name, booking_com_url,
+               star_rating, review_score, review_count,
+               tier, display_order, notes, first_seen_at, last_seen_at
+        FROM booking_com_hotels
+        WHERE is_active = TRUE
+    """
+    params = {}
+
+    if tier:
+        if tier not in ('own', 'competitor', 'market'):
+            raise HTTPException(status_code=400, detail="Invalid tier. Must be 'own', 'competitor', or 'market'")
+        query += " AND tier = :tier"
+        params['tier'] = tier
+
+    query += " ORDER BY display_order, name"
+
+    result = await db.execute(text(query), params)
+
+    return [
+        HotelResponse(
+            id=row.id,
+            booking_com_id=row.booking_com_id or '',
+            name=row.name,
+            booking_com_url=row.booking_com_url,
+            star_rating=float(row.star_rating) if row.star_rating else None,
+            review_score=float(row.review_score) if row.review_score else None,
+            review_count=row.review_count,
+            tier=row.tier,
+            display_order=row.display_order,
+            notes=row.notes,
+            first_seen_at=row.first_seen_at,
+            last_seen_at=row.last_seen_at
+        )
+        for row in result.fetchall()
+    ]
+
+
+@router.put("/hotels/{hotel_id}/tier")
+async def update_hotel_tier(
+    hotel_id: int,
+    update: HotelTierUpdate,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Update a hotel's tier and display order.
+
+    Tiers:
+    - 'own': Your hotel (for parity checking)
+    - 'competitor': Main competitors (full tracking)
+    - 'market': Other hotels (context only)
+    """
+    if update.tier not in ('own', 'competitor', 'market'):
+        raise HTTPException(status_code=400, detail="Invalid tier")
+
+    # If setting as 'own', clear any existing 'own' hotel
+    if update.tier == 'own':
+        await db.execute(
+            text("UPDATE booking_com_hotels SET tier = 'market' WHERE tier = 'own'")
+        )
+
+    # Update the hotel
+    set_clause = "tier = :tier"
+    params = {'hotel_id': hotel_id, 'tier': update.tier}
+
+    if update.display_order is not None:
+        set_clause += ", display_order = :order"
+        params['order'] = update.display_order
+
+    result = await db.execute(
+        text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id RETURNING id"),
+        params
+    )
+
+    if not result.fetchone():
+        raise HTTPException(status_code=404, detail="Hotel not found")
+
+    await db.commit()
+
+    # If this is now the own hotel, update system config
+    if update.tier == 'own':
+        await db.execute(
+            text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_own_hotel_id'"),
+            {'val': str(hotel_id)}
+        )
+        await db.commit()
+
+    return {"status": "success", "hotel_id": hotel_id, "tier": update.tier}
+
+
+@router.put("/hotels/{hotel_id}/notes")
+async def update_hotel_notes(
+    hotel_id: int,
+    notes: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Update notes for a hotel."""
+    result = await db.execute(
+        text("UPDATE booking_com_hotels SET notes = :notes WHERE id = :hotel_id RETURNING id"),
+        {'hotel_id': hotel_id, 'notes': notes}
+    )
+
+    if not result.fetchone():
+        raise HTTPException(status_code=404, detail="Hotel not found")
+
+    await db.commit()
+    return {"status": "success"}
+
+
+# ============================================
+# COMPETITOR RATES MATRIX
+# ============================================
+
+@router.get("/matrix")
+async def get_competitor_matrix(
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+    include_market: bool = False,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get rate comparison matrix for competitors.
+
+    Returns rates for own hotel and competitors, organized by date.
+    Set include_market=true to also include market tier hotels.
+    """
+    today = date.today()
+    start = date.fromisoformat(from_date) if from_date else today
+    end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
+
+    if end < start:
+        raise HTTPException(status_code=400, detail="to_date must be after from_date")
+    if (end - start).days > 90:
+        raise HTTPException(status_code=400, detail="Date range cannot exceed 90 days")
+
+    tier_filter = "h.tier IN ('own', 'competitor')"
+    if include_market:
+        tier_filter = "h.tier IN ('own', 'competitor', 'market')"
+
+    # Get hotels
+    hotels_result = await db.execute(
+        text(f"""
+            SELECT id, name, tier, display_order, star_rating, review_score, booking_com_url
+            FROM booking_com_hotels
+            WHERE is_active = TRUE AND {tier_filter.replace('h.', '')}
+            ORDER BY display_order, name
+        """)
+    )
+    hotels = [dict(row._mapping) for row in hotels_result.fetchall()]
+
+    # Get latest rates using the view
+    rates_result = await db.execute(
+        text(f"""
+            SELECT DISTINCT ON (r.hotel_id, r.rate_date)
+                r.hotel_id,
+                r.rate_date,
+                r.availability_status,
+                r.rate_gross,
+                r.room_type,
+                r.breakfast_included,
+                r.free_cancellation,
+                r.no_prepayment,
+                r.rooms_left,
+                r.scraped_at
+            FROM booking_com_rates r
+            JOIN booking_com_hotels h ON r.hotel_id = h.id
+            WHERE {tier_filter}
+              AND h.is_active = TRUE
+              AND r.rate_date >= :from_date AND r.rate_date <= :to_date
+            ORDER BY r.hotel_id, r.rate_date, r.scraped_at DESC
+        """),
+        {'from_date': start, 'to_date': end}
+    )
+
+    # Build matrix: hotel_id -> date -> rate data
+    rates_by_hotel: Dict[int, Dict[str, dict]] = {}
+    for row in rates_result.fetchall():
+        hotel_id = row.hotel_id
+        rate_date = row.rate_date.isoformat()
+
+        if hotel_id not in rates_by_hotel:
+            rates_by_hotel[hotel_id] = {}
+
+        rates_by_hotel[hotel_id][rate_date] = {
+            'availability_status': row.availability_status,
+            'rate_gross': float(row.rate_gross) if row.rate_gross else None,
+            'room_type': row.room_type,
+            'breakfast_included': row.breakfast_included,
+            'free_cancellation': row.free_cancellation,
+            'no_prepayment': row.no_prepayment,
+            'rooms_left': row.rooms_left,
+            'scraped_at': row.scraped_at.isoformat() if row.scraped_at else None,
+        }
+
+    # Build date list
+    dates = []
+    current = start
+    while current <= end:
+        dates.append(current.isoformat())
+        current += timedelta(days=1)
+
+    return {
+        'from_date': start.isoformat(),
+        'to_date': end.isoformat(),
+        'dates': dates,
+        'hotels': hotels,
+        'rates': rates_by_hotel
+    }
+
+
+# ============================================
+# RATE PARITY (OWN HOTEL VS NEWBOOK)
+# ============================================
+
+@router.get("/parity")
+async def get_rate_parity(
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get rate parity comparison between booking.com and Newbook rates.
+
+    Compares scraped booking.com rates for own hotel against Newbook current rates.
+    """
+    today = date.today()
+    start = date.fromisoformat(from_date) if from_date else today
+    end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
+
+    # Get own hotel's booking.com rates
+    booking_rates_result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (r.rate_date)
+                r.rate_date,
+                r.rate_gross as booking_rate,
+                r.availability_status,
+                r.room_type as booking_room_type,
+                r.scraped_at
+            FROM booking_com_rates r
+            JOIN booking_com_hotels h ON r.hotel_id = h.id
+            WHERE h.tier = 'own'
+              AND r.rate_date >= :from_date AND r.rate_date <= :to_date
+            ORDER BY r.rate_date, r.scraped_at DESC
+        """),
+        {'from_date': start, 'to_date': end}
+    )
+    booking_rates = {row.rate_date: dict(row._mapping) for row in booking_rates_result.fetchall()}
+
+    # Get Newbook rates (best rate per date across categories)
+    newbook_rates_result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (rate_date)
+                rate_date,
+                rate_gross as newbook_rate,
+                category_id
+            FROM newbook_current_rates
+            WHERE rate_date >= :from_date AND rate_date <= :to_date
+            ORDER BY rate_date, valid_from DESC
+        """),
+        {'from_date': start, 'to_date': end}
+    )
+    newbook_rates = {row.rate_date: dict(row._mapping) for row in newbook_rates_result.fetchall()}
+
+    # Compare rates
+    parity_issues = []
+    all_dates = set(booking_rates.keys()) | set(newbook_rates.keys())
+
+    for rate_date in sorted(all_dates):
+        booking = booking_rates.get(rate_date)
+        newbook = newbook_rates.get(rate_date)
+
+        if not booking or not newbook:
+            continue
+
+        booking_rate = booking.get('booking_rate')
+        newbook_rate = newbook.get('newbook_rate')
+
+        if not booking_rate or not newbook_rate:
+            continue
+
+        diff_pct = ((float(booking_rate) - float(newbook_rate)) / float(newbook_rate)) * 100
+
+        if abs(diff_pct) > 1:  # More than 1% difference
+            parity_issues.append({
+                'rate_date': rate_date.isoformat(),
+                'booking_rate': float(booking_rate),
+                'newbook_rate': float(newbook_rate),
+                'difference_pct': round(diff_pct, 2),
+                'alert_type': 'higher' if diff_pct > 0 else 'lower',
+                'booking_room_type': booking.get('booking_room_type'),
+                'availability_status': booking.get('availability_status'),
+            })
+
+    return {
+        'from_date': start.isoformat(),
+        'to_date': end.isoformat(),
+        'issues_count': len(parity_issues),
+        'issues': parity_issues
+    }
+
+
+# ============================================
+# PARITY ALERTS
+# ============================================
+
+@router.get("/parity/alerts")
+async def get_parity_alerts(
+    status: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get rate parity alerts."""
+    query = """
+        SELECT id, rate_date, room_category, newbook_rate, booking_com_rate,
+               difference_pct, alert_type, alert_status, created_at,
+               acknowledged_at, acknowledged_by, notes
+        FROM rate_parity_alerts
+    """
+    params = {}
+
+    if status:
+        query += " WHERE alert_status = :status"
+        params['status'] = status
+
+    query += " ORDER BY rate_date DESC, created_at DESC LIMIT 100"
+
+    result = await db.execute(text(query), params)
+
+    return [
+        {
+            'id': row.id,
+            'rate_date': row.rate_date.isoformat(),
+            'room_category': row.room_category,
+            'newbook_rate': float(row.newbook_rate) if row.newbook_rate else None,
+            'booking_com_rate': float(row.booking_com_rate) if row.booking_com_rate else None,
+            'difference_pct': float(row.difference_pct) if row.difference_pct else None,
+            'alert_type': row.alert_type,
+            'alert_status': row.alert_status,
+            'created_at': row.created_at.isoformat() if row.created_at else None,
+            'acknowledged_at': row.acknowledged_at.isoformat() if row.acknowledged_at else None,
+            'acknowledged_by': row.acknowledged_by,
+            'notes': row.notes,
+        }
+        for row in result.fetchall()
+    ]
+
+
+@router.put("/parity/alerts/{alert_id}/acknowledge")
+async def acknowledge_parity_alert(
+    alert_id: int,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Acknowledge a parity alert."""
+    result = await db.execute(
+        text("""
+            UPDATE rate_parity_alerts
+            SET alert_status = 'acknowledged',
+                acknowledged_at = NOW(),
+                acknowledged_by = :username
+            WHERE id = :alert_id
+            RETURNING id
+        """),
+        {'alert_id': alert_id, 'username': current_user.get('username', 'unknown')}
+    )
+
+    if not result.fetchone():
+        raise HTTPException(status_code=404, detail="Alert not found")
+
+    await db.commit()
+    return {"status": "success"}
+
+
+# ============================================
+# QUEUE STATUS
+# ============================================
+
+@router.get("/queue-status")
+async def get_queue_status(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get current scrape queue status."""
+    result = await db.execute(
+        text("""
+            SELECT
+                status,
+                COUNT(*) as count,
+                MIN(rate_date) as earliest_date,
+                MAX(rate_date) as latest_date
+            FROM booking_scrape_queue
+            GROUP BY status
+        """)
+    )
+    status_counts = {row.status: {
+        'count': row.count,
+        'earliest': row.earliest_date.isoformat() if row.earliest_date else None,
+        'latest': row.latest_date.isoformat() if row.latest_date else None,
+    } for row in result.fetchall()}
+
+    # Get retry items (failed but under max_attempts)
+    retry_result = await db.execute(
+        text("""
+            SELECT COUNT(*) as count
+            FROM booking_scrape_queue
+            WHERE status = 'pending' AND attempts > 0
+        """)
+    )
+    retry_count = retry_result.fetchone().count
+
+    return {
+        'statuses': status_counts,
+        'retries_pending': retry_count,
+        'total_pending': status_counts.get('pending', {}).get('count', 0),
+        'total_completed': status_counts.get('completed', {}).get('count', 0),
+        'total_failed': status_counts.get('failed', {}).get('count', 0),
+    }
+
+
+# ============================================
+# SCHEDULE INFO
+# ============================================
+
+@router.get("/schedule-info")
+async def get_schedule_info(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get information about the scraping schedule."""
+    # Get configured time
+    time_result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_daily_time'")
+    )
+    time_row = time_result.fetchone()
+    daily_time = time_row.config_value if time_row and time_row.config_value else '05:30'
+
+    # Calculate what today's schedule would look like
+    from jobs.scrape_booking_rates import get_high_priority_dates, get_medium_priority_dates, get_low_priority_dates
+    high = get_high_priority_dates()
+    medium = get_medium_priority_dates()
+    low = get_low_priority_dates()
+
+    today = date.today()
+    weekday_name = today.strftime('%A')
+
+    return {
+        'daily_time': daily_time,
+        'today': today.isoformat(),
+        'weekday': weekday_name,
+        'tiers': {
+            'high': {
+                'description': 'Next 30 days (scraped first)',
+                'dates_today': len(high),
+                'range': f'{high[0].isoformat()} to {high[-1].isoformat()}' if high else None,
+            },
+            'medium': {
+                'description': 'Days 31-180 (scraped after high priority)',
+                'dates_today': len(medium),
+                'range': f'{medium[0].isoformat()} to {medium[-1].isoformat()}' if medium else None,
+            },
+            'low': {
+                'description': 'Days 181-365 (scraped last, or until rate limit)',
+                'dates_today': len(low),
+                'range': f'{low[0].isoformat()} to {low[-1].isoformat()}' if low else None,
+            },
+        },
+        'total_dates_today': len(set(high + medium + low)),
+    }
+
+
+# ============================================
+# SCRAPE COVERAGE (365-day view)
+# ============================================
+
+@router.get("/scrape-coverage")
+async def get_scrape_coverage(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get 365-day scrape coverage showing last scraped time
+    and next expected scrape for every date.
+    """
+    today = date.today()
+    end = today + timedelta(days=365)
+
+    # Get latest scraped_at per date (across all hotels)
+    result = await db.execute(
+        text("""
+            SELECT rate_date, MAX(scraped_at) as last_scraped
+            FROM booking_com_rates
+            WHERE rate_date >= :from_date AND rate_date <= :to_date
+            GROUP BY rate_date
+        """),
+        {'from_date': today, 'to_date': end}
+    )
+    scraped_map = {row.rate_date: row.last_scraped for row in result.fetchall()}
+
+    # Compute tier and next scrape for each date
+    from jobs.scrape_booking_rates import compute_next_scrape_for_date
+
+    coverage = []
+    for offset in range(366):
+        d = today + timedelta(days=offset)
+        tier, next_scrape = compute_next_scrape_for_date(d)
+        last_scraped = scraped_map.get(d)
+
+        coverage.append({
+            'date': d.isoformat(),
+            'tier': tier,
+            'last_scraped': last_scraped.isoformat() if last_scraped else None,
+            'next_expected': next_scrape.isoformat() if next_scrape else None,
+        })
+
+    return {
+        'today': today.isoformat(),
+        'coverage': coverage,
+    }
+
+
+# ============================================
+# BOOKING.COM AVAILABILITY CHECK (for Bookability page)
+# ============================================
+
+@router.get("/booking-availability")
+async def get_booking_availability(
+    from_date: Optional[str] = None,
+    to_date: Optional[str] = None,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Check own hotel's availability on booking.com.
+
+    Returns a simple summary: for each date in the range, whether the own hotel
+    appears available on booking.com based on the latest scrape data.
+    """
+    today = date.today()
+    start = date.fromisoformat(from_date) if from_date else today
+    end = date.fromisoformat(to_date) if to_date else today + timedelta(days=30)
+
+    # Get own hotel's latest scraped availability
+    result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (r.rate_date)
+                r.rate_date,
+                r.availability_status,
+                r.rate_gross,
+                r.scraped_at
+            FROM booking_com_rates r
+            JOIN booking_com_hotels h ON r.hotel_id = h.id
+            WHERE h.tier = 'own'
+              AND r.rate_date >= :from_date AND r.rate_date <= :to_date
+            ORDER BY r.rate_date, r.scraped_at DESC
+        """),
+        {'from_date': start, 'to_date': end}
+    )
+    rows = result.fetchall()
+
+    if not rows:
+        return {
+            'has_own_hotel': False,
+            'dates_checked': 0,
+            'dates_available': 0,
+            'dates_sold_out': 0,
+            'dates_no_data': 0,
+            'latest_scrape': None,
+            'dates': {},
+        }
+
+    dates_map = {}
+    dates_available = 0
+    dates_sold_out = 0
+    dates_no_data = 0
+    latest_scrape = None
+
+    for row in rows:
+        status = row.availability_status
+        dates_map[row.rate_date.isoformat()] = {
+            'status': status,
+            'rate': float(row.rate_gross) if row.rate_gross else None,
+        }
+        if status == 'available':
+            dates_available += 1
+        elif status == 'sold_out':
+            dates_sold_out += 1
+        else:
+            dates_no_data += 1
+
+        if row.scraped_at and (not latest_scrape or row.scraped_at > latest_scrape):
+            latest_scrape = row.scraped_at
+
+    return {
+        'has_own_hotel': True,
+        'dates_checked': len(rows),
+        'dates_available': dates_available,
+        'dates_sold_out': dates_sold_out,
+        'dates_no_data': dates_no_data,
+        'latest_scrape': latest_scrape.isoformat() if latest_scrape else None,
+        'dates': dates_map,
+    }
+
+
+# ============================================
+# SCRAPE HISTORY
+# ============================================
+
+@router.get("/scrape-history")
+async def get_scrape_history(
+    limit: int = 20,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get recent scrape batch history."""
+    result = await db.execute(
+        text("""
+            SELECT batch_id, scrape_type, started_at, completed_at, status,
+                   dates_queued, dates_completed, dates_failed,
+                   hotels_found, rates_scraped, error_message,
+                   blocked_at, resume_after
+            FROM booking_scrape_log
+            ORDER BY started_at DESC
+            LIMIT :limit
+        """),
+        {'limit': limit}
+    )
+
+    return [
+        {
+            'batch_id': str(row.batch_id),
+            'scrape_type': row.scrape_type,
+            'started_at': row.started_at.isoformat() if row.started_at else None,
+            'completed_at': row.completed_at.isoformat() if row.completed_at else None,
+            'status': row.status,
+            'dates_queued': row.dates_queued,
+            'dates_completed': row.dates_completed,
+            'dates_failed': row.dates_failed,
+            'hotels_found': row.hotels_found,
+            'rates_scraped': row.rates_scraped,
+            'error_message': row.error_message,
+            'blocked_at': row.blocked_at.isoformat() if row.blocked_at else None,
+            'resume_after': row.resume_after.isoformat() if row.resume_after else None,
+        }
+        for row in result.fetchall()
+    ]

backend/api/crossref.py

@@ -0,0 +1,300 @@
+"""
+Cross-Reference Validation API endpoints
+Validate that related forecasts align with each other
+"""
+from datetime import date
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+@router.get("/check")
+async def run_cross_reference_check(
+    check_date: date = Query(..., description="Date to run cross-reference checks for"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Run all cross-reference checks for a specific date.
+    Returns validation results showing if forecasts are internally consistent.
+    """
+    # Get all active cross-reference configurations
+    config_query = """
+        SELECT check_name, check_category, formula, compares_to, tolerance_pct, input_metrics
+        FROM cross_reference_config
+        WHERE is_active = TRUE
+        ORDER BY display_order
+    """
+
+    config_result = await db.execute(text(config_query))
+    configs = config_result.fetchall()
+
+    # Get all forecast values for the date
+    forecast_query = """
+        SELECT forecast_type, predicted_value
+        FROM forecasts
+        WHERE forecast_date = :check_date
+            AND model_type = 'prophet'
+    """
+
+    forecast_result = await db.execute(text(forecast_query), {"check_date": check_date})
+    forecasts = {row.forecast_type: float(row.predicted_value) for row in forecast_result.fetchall()}
+
+    results = []
+    for config in configs:
+        # For now, return placeholder results
+        # In production, would evaluate formula against forecasts
+        compares_to_value = forecasts.get(config.compares_to)
+
+        results.append({
+            "check_name": config.check_name,
+            "check_category": config.check_category,
+            "formula": config.formula,
+            "compares_to": config.compares_to,
+            "forecasted_value": compares_to_value,
+            "calculated_value": None,  # Would be calculated from formula
+            "difference": None,
+            "difference_pct": None,
+            "tolerance_pct": float(config.tolerance_pct),
+            "status": "ok"  # Would be evaluated based on tolerance
+        })
+
+    return {
+        "check_date": check_date,
+        "results": results,
+        "alignment_score": 100  # Would be calculated
+    }
+
+
+@router.get("/report")
+async def get_cross_reference_report(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get cross-reference validation report for date range.
+    """
+    query = """
+        SELECT
+            forecast_date,
+            check_name,
+            check_category,
+            calculated_value,
+            forecasted_value,
+            difference,
+            difference_pct,
+            tolerance_pct,
+            status
+        FROM forecast_cross_reference
+        WHERE forecast_date BETWEEN :from_date AND :to_date
+        ORDER BY forecast_date, check_category, check_name
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.forecast_date,
+            "check_name": row.check_name,
+            "category": row.check_category,
+            "calculated": float(row.calculated_value) if row.calculated_value else None,
+            "forecasted": float(row.forecasted_value) if row.forecasted_value else None,
+            "difference": float(row.difference) if row.difference else None,
+            "difference_pct": float(row.difference_pct) if row.difference_pct else None,
+            "tolerance_pct": float(row.tolerance_pct),
+            "status": row.status
+        }
+        for row in rows
+    ]
+
+
+@router.get("/discrepancies")
+async def get_discrepancies(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get list of dates with cross-reference discrepancies.
+    """
+    query = """
+        SELECT
+            forecast_date,
+            check_name,
+            check_category,
+            difference_pct,
+            tolerance_pct,
+            possible_causes,
+            recommendation
+        FROM forecast_cross_reference
+        WHERE forecast_date BETWEEN :from_date AND :to_date
+            AND status = 'discrepancy'
+        ORDER BY ABS(difference_pct) DESC
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.forecast_date,
+            "check_name": row.check_name,
+            "category": row.check_category,
+            "difference_pct": float(row.difference_pct) if row.difference_pct else None,
+            "tolerance_pct": float(row.tolerance_pct),
+            "possible_causes": row.possible_causes,
+            "recommendation": row.recommendation
+        }
+        for row in rows
+    ]
+
+
+@router.get("/alignment-score")
+async def get_alignment_score(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get overall alignment score for date range.
+    Higher score = more internally consistent forecasts.
+    """
+    query = """
+        SELECT
+            COUNT(*) as total_checks,
+            SUM(CASE WHEN status = 'ok' THEN 1 ELSE 0 END) as passed,
+            SUM(CASE WHEN status = 'warning' THEN 1 ELSE 0 END) as warnings,
+            SUM(CASE WHEN status = 'discrepancy' THEN 1 ELSE 0 END) as discrepancies
+        FROM forecast_cross_reference
+        WHERE forecast_date BETWEEN :from_date AND :to_date
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    row = result.fetchone()
+
+    if not row or row.total_checks == 0:
+        return {
+            "from_date": from_date,
+            "to_date": to_date,
+            "alignment_score": 100,
+            "total_checks": 0,
+            "passed": 0,
+            "warnings": 0,
+            "discrepancies": 0
+        }
+
+    # Score: 100 * (passed / total), with warnings counting as 0.5
+    score = ((row.passed + row.warnings * 0.5) / row.total_checks) * 100
+
+    return {
+        "from_date": from_date,
+        "to_date": to_date,
+        "alignment_score": round(score, 1),
+        "total_checks": row.total_checks,
+        "passed": row.passed,
+        "warnings": row.warnings,
+        "discrepancies": row.discrepancies
+    }
+
+
+@router.get("/config")
+async def get_crossref_config(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get cross-reference check configuration.
+    """
+    query = """
+        SELECT
+            check_name,
+            check_category,
+            description,
+            formula,
+            compares_to,
+            tolerance_pct,
+            input_metrics,
+            is_correlation_check,
+            expected_correlation,
+            is_active,
+            display_order
+        FROM cross_reference_config
+        ORDER BY display_order
+    """
+
+    result = await db.execute(text(query))
+    rows = result.fetchall()
+
+    return [
+        {
+            "check_name": row.check_name,
+            "category": row.check_category,
+            "description": row.description,
+            "formula": row.formula,
+            "compares_to": row.compares_to,
+            "tolerance_pct": float(row.tolerance_pct),
+            "input_metrics": row.input_metrics,
+            "is_correlation_check": row.is_correlation_check,
+            "expected_correlation": float(row.expected_correlation) if row.expected_correlation else None,
+            "is_active": row.is_active
+        }
+        for row in rows
+    ]
+
+
+@router.put("/config/{check_name}")
+async def update_crossref_config(
+    check_name: str,
+    tolerance_pct: Optional[float] = Query(None),
+    is_active: Optional[bool] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Update tolerance or active status for a cross-reference check.
+    """
+    updates = []
+    params = {"check_name": check_name}
+
+    if tolerance_pct is not None:
+        updates.append("tolerance_pct = :tolerance_pct")
+        params["tolerance_pct"] = tolerance_pct
+
+    if is_active is not None:
+        updates.append("is_active = :is_active")
+        params["is_active"] = is_active
+
+    if not updates:
+        return {"status": "no_changes", "check_name": check_name}
+
+    query = f"""
+        UPDATE cross_reference_config
+        SET {', '.join(updates)}
+        WHERE check_name = :check_name
+        RETURNING check_name
+    """
+
+    result = await db.execute(text(query), params)
+    await db.commit()
+
+    row = result.fetchone()
+    if not row:
+        raise ValueError(f"Check not found: {check_name}")
+
+    return {
+        "status": "updated",
+        "check_name": check_name,
+        "updates": {k: v for k, v in params.items() if k != "check_name"}
+    }

backend/api/evolution.py

@@ -0,0 +1,292 @@
+"""
+Forecast Evolution API endpoints
+Track how forecasts change over time as dates approach
+"""
+from datetime import date, timedelta
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+@router.get("/date")
+async def get_forecast_evolution_for_date(
+    forecast_date: date = Query(..., description="The date to see evolution for"),
+    forecast_type: str = Query(..., description="Metric code"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get full forecast history for a specific date.
+    Shows how predictions changed as the date approached.
+    """
+    query = """
+        SELECT
+            fh.generated_at,
+            fh.model_type,
+            fh.predicted_value,
+            fh.lower_bound,
+            fh.upper_bound,
+            fh.horizon_days,
+            fh.change_amount,
+            fh.change_pct,
+            fh.change_reason,
+            dm.actual_value
+        FROM forecast_history fh
+        LEFT JOIN daily_metrics dm ON fh.forecast_date = dm.date AND fh.forecast_type = dm.metric_code
+        WHERE fh.forecast_date = :forecast_date
+            AND fh.forecast_type = :forecast_type
+        ORDER BY fh.generated_at, fh.model_type
+    """
+
+    result = await db.execute(text(query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type
+    })
+    rows = result.fetchall()
+
+    return [
+        {
+            "generated_at": row.generated_at,
+            "model_type": row.model_type,
+            "predicted_value": float(row.predicted_value),
+            "lower_bound": float(row.lower_bound) if row.lower_bound else None,
+            "upper_bound": float(row.upper_bound) if row.upper_bound else None,
+            "horizon_days": row.horizon_days,
+            "change_amount": float(row.change_amount) if row.change_amount else None,
+            "change_pct": float(row.change_pct) if row.change_pct else None,
+            "change_reason": row.change_reason,
+            "actual_value": float(row.actual_value) if row.actual_value else None
+        }
+        for row in rows
+    ]
+
+
+@router.get("/chart-data")
+async def get_evolution_chart_data(
+    forecast_date: date = Query(...),
+    forecast_type: str = Query(...),
+    model: str = Query("prophet", description="Model: prophet, xgboost, pickup"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get evolution data formatted for charting.
+    Returns time series of forecast values as date approached.
+    """
+    query = """
+        SELECT
+            DATE(fh.generated_at) as update_date,
+            fh.horizon_days,
+            fh.predicted_value,
+            fh.lower_bound,
+            fh.upper_bound,
+            dm.actual_value
+        FROM forecast_history fh
+        LEFT JOIN daily_metrics dm ON fh.forecast_date = dm.date AND fh.forecast_type = dm.metric_code
+        WHERE fh.forecast_date = :forecast_date
+            AND fh.forecast_type = :forecast_type
+            AND fh.model_type = :model
+        ORDER BY fh.generated_at
+    """
+
+    result = await db.execute(text(query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type,
+        "model": model
+    })
+    rows = result.fetchall()
+
+    actual_value = None
+    chart_data = []
+
+    for row in rows:
+        # Use 'is not None' - 0 is valid actual data (e.g., 0 covers on closed day)
+        if row.actual_value is not None:
+            actual_value = float(row.actual_value)
+        chart_data.append({
+            "update_date": row.update_date,
+            "horizon_days": row.horizon_days,
+            "predicted_value": float(row.predicted_value),
+            "lower_bound": float(row.lower_bound) if row.lower_bound else None,
+            "upper_bound": float(row.upper_bound) if row.upper_bound else None
+        })
+
+    return {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type,
+        "model": model,
+        "actual_value": actual_value,
+        "data_points": chart_data
+    }
+
+
+@router.get("/changes")
+async def get_forecast_changes(
+    forecast_date: date = Query(...),
+    forecast_type: str = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get list of all changes with reasons for a specific forecast.
+    """
+    query = """
+        SELECT
+            changed_at,
+            model_type,
+            old_value,
+            new_value,
+            change_amount,
+            change_pct,
+            change_category,
+            change_reason,
+            bookings_added,
+            bookings_cancelled,
+            covers_change,
+            days_out,
+            otb_at_change
+        FROM forecast_change_log
+        WHERE forecast_date = :forecast_date
+            AND forecast_type = :forecast_type
+        ORDER BY changed_at DESC
+    """
+
+    result = await db.execute(text(query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type
+    })
+    rows = result.fetchall()
+
+    return [
+        {
+            "changed_at": row.changed_at,
+            "model_type": row.model_type,
+            "old_value": float(row.old_value) if row.old_value else None,
+            "new_value": float(row.new_value) if row.new_value else None,
+            "change_amount": float(row.change_amount) if row.change_amount else None,
+            "change_pct": float(row.change_pct) if row.change_pct else None,
+            "change_category": row.change_category,
+            "change_reason": row.change_reason,
+            "bookings_added": row.bookings_added,
+            "bookings_cancelled": row.bookings_cancelled,
+            "covers_change": row.covers_change,
+            "days_out": row.days_out,
+            "otb_at_change": float(row.otb_at_change) if row.otb_at_change is not None else None
+        }
+        for row in rows
+    ]
+
+
+@router.get("/convergence")
+async def get_forecast_convergence(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Analyze how quickly forecasts converge to actuals.
+    Shows forecast accuracy at different lead times.
+    """
+    query = """
+        WITH convergence_data AS (
+            SELECT
+                fh.forecast_type,
+                fh.model_type,
+                fh.horizon_days,
+                ABS(fh.predicted_value - dm.actual_value) as abs_error,
+                ABS((fh.predicted_value - dm.actual_value) / NULLIF(dm.actual_value, 0) * 100) as pct_error
+            FROM forecast_history fh
+            JOIN daily_metrics dm ON fh.forecast_date = dm.date AND fh.forecast_type = dm.metric_code
+            WHERE fh.forecast_date BETWEEN :from_date AND :to_date
+                AND dm.actual_value IS NOT NULL
+        )
+        SELECT
+            forecast_type,
+            model_type,
+            CASE
+                WHEN horizon_days <= 7 THEN '0-7 days'
+                WHEN horizon_days <= 14 THEN '8-14 days'
+                WHEN horizon_days <= 21 THEN '15-21 days'
+                WHEN horizon_days <= 28 THEN '22-28 days'
+                ELSE '29+ days'
+            END as horizon_bucket,
+            AVG(abs_error) as avg_error,
+            AVG(pct_error) as avg_pct_error,
+            COUNT(*) as sample_count
+        FROM convergence_data
+        GROUP BY forecast_type, model_type,
+            CASE
+                WHEN horizon_days <= 7 THEN '0-7 days'
+                WHEN horizon_days <= 14 THEN '8-14 days'
+                WHEN horizon_days <= 21 THEN '15-21 days'
+                WHEN horizon_days <= 28 THEN '22-28 days'
+                ELSE '29+ days'
+            END
+        ORDER BY forecast_type, model_type, horizon_bucket
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "forecast_type": row.forecast_type,
+            "model_type": row.model_type,
+            "horizon_bucket": row.horizon_bucket,
+            "avg_error": round(float(row.avg_error), 2) if row.avg_error else None,
+            "avg_pct_error": round(float(row.avg_pct_error), 2) if row.avg_pct_error else None,
+            "sample_count": row.sample_count
+        }
+        for row in rows
+    ]
+
+
+@router.get("/volatility")
+async def get_forecast_volatility(
+    from_date: date = Query(...),
+    to_date: date = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Identify dates with high forecast volatility.
+    Shows which dates had the most forecast changes.
+    """
+    query = """
+        SELECT
+            forecast_date,
+            forecast_type,
+            COUNT(*) as change_count,
+            MAX(ABS(change_amount)) as max_change,
+            SUM(ABS(change_amount)) as total_change,
+            array_agg(DISTINCT change_category) as change_categories
+        FROM forecast_change_log
+        WHERE forecast_date BETWEEN :from_date AND :to_date
+        GROUP BY forecast_date, forecast_type
+        HAVING COUNT(*) > 3 OR MAX(ABS(change_pct)) > 10
+        ORDER BY total_change DESC
+        LIMIT 20
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "forecast_date": row.forecast_date,
+            "forecast_type": row.forecast_type,
+            "change_count": row.change_count,
+            "max_change": float(row.max_change) if row.max_change else None,
+            "total_change": float(row.total_change) if row.total_change else None,
+            "change_categories": row.change_categories
+        }
+        for row in rows
+    ]

backend/api/explain.py

@@ -0,0 +1,328 @@
+"""
+Model Explainability API endpoints
+Explain why forecasts have specific values
+"""
+from datetime import date
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+@router.get("/forecast")
+async def explain_forecast(
+    forecast_date: date = Query(...),
+    forecast_type: str = Query(...),
+    model: str = Query("prophet", description="Model: prophet, xgboost, pickup"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get full breakdown of how a forecast was calculated.
+    Returns different explanations based on model type.
+    """
+    # Get the forecast value
+    forecast_query = """
+        SELECT predicted_value, lower_bound, upper_bound, generated_at
+        FROM forecasts
+        WHERE forecast_date = :forecast_date
+            AND forecast_type = :forecast_type
+            AND model_type = :model
+        ORDER BY generated_at DESC
+        LIMIT 1
+    """
+
+    result = await db.execute(text(forecast_query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type,
+        "model": model
+    })
+    forecast = result.fetchone()
+
+    if not forecast:
+        return {
+            "forecast_date": forecast_date,
+            "forecast_type": forecast_type,
+            "model": model,
+            "error": "No forecast found"
+        }
+
+    # Get model-specific explanation
+    if model == "prophet":
+        return await _get_prophet_explanation(db, forecast_date, forecast_type, forecast)
+    elif model == "xgboost":
+        return await _get_xgboost_explanation(db, forecast_date, forecast_type, forecast)
+    elif model == "pickup":
+        return await _get_pickup_explanation(db, forecast_date, forecast_type, forecast)
+    else:
+        return {
+            "forecast_date": forecast_date,
+            "forecast_type": forecast_type,
+            "model": model,
+            "predicted_value": float(forecast.predicted_value),
+            "explanation": "No detailed explanation available for this model"
+        }
+
+
+async def _get_prophet_explanation(db, forecast_date, forecast_type, forecast):
+    """Get Prophet model decomposition explanation"""
+    query = """
+        SELECT
+            trend,
+            yearly_seasonality,
+            weekly_seasonality,
+            daily_seasonality,
+            holiday_effects,
+            regressor_effects
+        FROM prophet_decomposition
+        WHERE forecast_date = :forecast_date
+            AND forecast_type = :forecast_type
+        ORDER BY generated_at DESC
+        LIMIT 1
+    """
+
+    result = await db.execute(text(query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type
+    })
+    decomp = result.fetchone()
+
+    explanation = {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type,
+        "model": "prophet",
+        "predicted_value": float(forecast.predicted_value),
+        "lower_bound": float(forecast.lower_bound) if forecast.lower_bound else None,
+        "upper_bound": float(forecast.upper_bound) if forecast.upper_bound else None,
+        "generated_at": forecast.generated_at
+    }
+
+    if decomp:
+        components = []
+        if decomp.trend:
+            components.append({
+                "name": "Base trend",
+                "value": float(decomp.trend),
+                "description": "Long-term average trend"
+            })
+        if decomp.yearly_seasonality:
+            components.append({
+                "name": "Yearly seasonality",
+                "value": float(decomp.yearly_seasonality),
+                "description": "Annual pattern (e.g., summer peak, winter low)"
+            })
+        if decomp.weekly_seasonality:
+            components.append({
+                "name": "Weekly seasonality",
+                "value": float(decomp.weekly_seasonality),
+                "description": "Day-of-week pattern (e.g., weekend higher)"
+            })
+        if decomp.holiday_effects:
+            for holiday, effect in decomp.holiday_effects.items():
+                components.append({
+                    "name": f"Holiday: {holiday}",
+                    "value": float(effect),
+                    "description": f"Effect of {holiday}"
+                })
+
+        explanation["components"] = components
+        explanation["breakdown"] = {
+            "trend": float(decomp.trend) if decomp.trend else 0,
+            "yearly": float(decomp.yearly_seasonality) if decomp.yearly_seasonality else 0,
+            "weekly": float(decomp.weekly_seasonality) if decomp.weekly_seasonality else 0,
+            "holidays": decomp.holiday_effects
+        }
+
+    return explanation
+
+
+async def _get_xgboost_explanation(db, forecast_date, forecast_type, forecast):
+    """Get XGBoost SHAP explanation"""
+    query = """
+        SELECT
+            base_value,
+            feature_values,
+            shap_values,
+            top_positive,
+            top_negative
+        FROM xgboost_explanations
+        WHERE forecast_date = :forecast_date
+            AND forecast_type = :forecast_type
+        ORDER BY generated_at DESC
+        LIMIT 1
+    """
+
+    result = await db.execute(text(query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type
+    })
+    shap = result.fetchone()
+
+    explanation = {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type,
+        "model": "xgboost",
+        "predicted_value": float(forecast.predicted_value),
+        "generated_at": forecast.generated_at
+    }
+
+    if shap:
+        explanation["base_value"] = float(shap.base_value) if shap.base_value else None
+        explanation["feature_values"] = shap.feature_values
+        explanation["shap_values"] = shap.shap_values
+        explanation["top_drivers"] = {
+            "positive": shap.top_positive or [],
+            "negative": shap.top_negative or []
+        }
+
+        # Build human-readable summary
+        summary_parts = []
+        if shap.top_positive:
+            for item in shap.top_positive[:3]:
+                if isinstance(item, dict):
+                    summary_parts.append(f"{item.get('feature', 'Unknown')} (+{item.get('contribution', 0):.1f})")
+        if shap.top_negative:
+            for item in shap.top_negative[:2]:
+                if isinstance(item, dict):
+                    summary_parts.append(f"{item.get('feature', 'Unknown')} ({item.get('contribution', 0):.1f})")
+
+        explanation["summary"] = f"Main drivers: {', '.join(summary_parts)}" if summary_parts else None
+
+    return explanation
+
+
+async def _get_pickup_explanation(db, forecast_date, forecast_type, forecast):
+    """Get Pickup model explanation"""
+    query = """
+        SELECT
+            current_otb,
+            days_out,
+            comparison_date,
+            comparison_otb,
+            comparison_final,
+            pickup_curve_pct,
+            pickup_curve_stddev,
+            pace_vs_prior_pct,
+            projection_method,
+            projected_value,
+            confidence_note
+        FROM pickup_explanations
+        WHERE forecast_date = :forecast_date
+            AND forecast_type = :forecast_type
+        ORDER BY generated_at DESC
+        LIMIT 1
+    """
+
+    result = await db.execute(text(query), {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type
+    })
+    pickup = result.fetchone()
+
+    explanation = {
+        "forecast_date": forecast_date,
+        "forecast_type": forecast_type,
+        "model": "pickup",
+        "predicted_value": float(forecast.predicted_value),
+        "generated_at": forecast.generated_at
+    }
+
+    if pickup:
+        explanation["current_state"] = {
+            "on_the_books": float(pickup.current_otb) if pickup.current_otb is not None else None,
+            "days_out": pickup.days_out
+        }
+        explanation["comparison"] = {
+            "date": pickup.comparison_date,
+            "otb_at_same_lead_time": float(pickup.comparison_otb) if pickup.comparison_otb is not None else None,
+            "final_actual": float(pickup.comparison_final) if pickup.comparison_final is not None else None
+        }
+        explanation["pickup_curve"] = {
+            "avg_pct_of_final": float(pickup.pickup_curve_pct) if pickup.pickup_curve_pct is not None else None,
+            "std_dev": float(pickup.pickup_curve_stddev) if pickup.pickup_curve_stddev is not None else None
+        }
+        explanation["pace_analysis"] = {
+            "vs_prior_year_pct": float(pickup.pace_vs_prior_pct) if pickup.pace_vs_prior_pct is not None else None,
+            "projection_method": pickup.projection_method,
+            "projected_final": float(pickup.projected_value) if pickup.projected_value else None
+        }
+        explanation["confidence_note"] = pickup.confidence_note
+
+        # Build summary
+        pace_str = ""
+        if pickup.pace_vs_prior_pct:
+            if pickup.pace_vs_prior_pct > 0:
+                pace_str = f"{pickup.pace_vs_prior_pct:.1f}% ahead of last year's pace"
+            else:
+                pace_str = f"{abs(pickup.pace_vs_prior_pct):.1f}% behind last year's pace"
+
+        explanation["summary"] = f"At {pickup.days_out} days out, {pace_str}" if pace_str else None
+
+    return explanation
+
+
+@router.get("/prophet")
+async def get_prophet_decomposition(
+    forecast_date: date = Query(...),
+    forecast_type: str = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get Prophet decomposition (trend, seasonality, holidays) for a forecast.
+    """
+    return await _get_prophet_explanation(
+        db, forecast_date, forecast_type,
+        type('obj', (object,), {
+            'predicted_value': 0,
+            'lower_bound': None,
+            'upper_bound': None,
+            'generated_at': None
+        })()
+    )
+
+
+@router.get("/xgboost")
+async def get_xgboost_shap(
+    forecast_date: date = Query(...),
+    forecast_type: str = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get XGBoost SHAP values and feature contributions for a forecast.
+    """
+    return await _get_xgboost_explanation(
+        db, forecast_date, forecast_type,
+        type('obj', (object,), {
+            'predicted_value': 0,
+            'generated_at': None
+        })()
+    )
+
+
+@router.get("/pickup")
+async def get_pickup_breakdown(
+    forecast_date: date = Query(...),
+    forecast_type: str = Query(...),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get Pickup model calculation breakdown for a forecast.
+    """
+    return await _get_pickup_explanation(
+        db, forecast_date, forecast_type,
+        type('obj', (object,), {
+            'predicted_value': 0,
+            'generated_at': None
+        })()
+    )
+
+

backend/api/export.py

@@ -0,0 +1,237 @@
+"""
+Export API endpoints for Excel/CSV downloads
+"""
+import io
+from datetime import date, timedelta
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from fastapi.responses import StreamingResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+import pandas as pd
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+@router.get("/excel")
+async def export_excel(
+    from_date: Optional[date] = Query(None),
+    to_date: Optional[date] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Download Excel workbook with multiple sheets:
+    - Daily Forecast (all models)
+    - Weekly Summary
+    - Budget Comparison
+    - Model Accuracy
+    """
+    if from_date is None:
+        from_date = date.today()
+    if to_date is None:
+        to_date = from_date + timedelta(days=28)
+
+    # Create Excel writer
+    output = io.BytesIO()
+    with pd.ExcelWriter(output, engine='openpyxl') as writer:
+        # Daily forecasts sheet
+        daily_query = """
+            SELECT
+                f.forecast_date as "Date",
+                f.forecast_type as "Metric",
+                MAX(CASE WHEN f.model_type = 'prophet' THEN f.predicted_value END) as "Prophet",
+                MAX(CASE WHEN f.model_type = 'prophet' THEN f.lower_bound END) as "Prophet Lower",
+                MAX(CASE WHEN f.model_type = 'prophet' THEN f.upper_bound END) as "Prophet Upper",
+                MAX(CASE WHEN f.model_type = 'xgboost' THEN f.predicted_value END) as "XGBoost",
+                MAX(CASE WHEN f.model_type = 'pickup' THEN f.predicted_value END) as "Pickup",
+                db.budget_value as "Budget"
+            FROM forecasts f
+            LEFT JOIN daily_budgets db ON f.forecast_date = db.date AND f.forecast_type = db.budget_type
+            WHERE f.forecast_date BETWEEN :from_date AND :to_date
+            GROUP BY f.forecast_date, f.forecast_type, db.budget_value
+            ORDER BY f.forecast_date, f.forecast_type
+        """
+        result = await db.execute(text(daily_query), {"from_date": from_date, "to_date": to_date})
+        daily_df = pd.DataFrame(result.fetchall())
+        if not daily_df.empty:
+            daily_df.to_excel(writer, sheet_name='Daily Forecast', index=False)
+
+        # Weekly summary sheet
+        weekly_query = """
+            SELECT
+                DATE_TRUNC('week', f.forecast_date) as "Week Start",
+                f.forecast_type as "Metric",
+                AVG(f.predicted_value) as "Avg Forecast",
+                SUM(f.predicted_value) as "Total Forecast",
+                AVG(db.budget_value) as "Avg Budget",
+                SUM(db.budget_value) as "Total Budget"
+            FROM forecasts f
+            LEFT JOIN daily_budgets db ON f.forecast_date = db.date AND f.forecast_type = db.budget_type
+            WHERE f.forecast_date BETWEEN :from_date AND :to_date
+                AND f.model_type = 'prophet'
+            GROUP BY DATE_TRUNC('week', f.forecast_date), f.forecast_type
+            ORDER BY "Week Start", f.forecast_type
+        """
+        result = await db.execute(text(weekly_query), {"from_date": from_date, "to_date": to_date})
+        weekly_df = pd.DataFrame(result.fetchall())
+        if not weekly_df.empty:
+            weekly_df.to_excel(writer, sheet_name='Weekly Summary', index=False)
+
+        # Budget variance sheet
+        variance_query = """
+            SELECT
+                f.forecast_date as "Date",
+                f.forecast_type as "Metric",
+                f.predicted_value as "Forecast",
+                db.budget_value as "Budget",
+                (f.predicted_value - db.budget_value) as "Variance",
+                CASE
+                    WHEN db.budget_value != 0 THEN
+                        ROUND(((f.predicted_value - db.budget_value) / db.budget_value * 100)::numeric, 1)
+                    ELSE NULL
+                END as "Variance %"
+            FROM forecasts f
+            LEFT JOIN daily_budgets db ON f.forecast_date = db.date AND f.forecast_type = db.budget_type
+            WHERE f.forecast_date BETWEEN :from_date AND :to_date
+                AND f.model_type = 'prophet'
+            ORDER BY f.forecast_date, f.forecast_type
+        """
+        result = await db.execute(text(variance_query), {"from_date": from_date, "to_date": to_date})
+        variance_df = pd.DataFrame(result.fetchall())
+        if not variance_df.empty:
+            variance_df.to_excel(writer, sheet_name='Budget Variance', index=False)
+
+        # Accuracy sheet (historical)
+        accuracy_query = """
+            SELECT
+                date as "Date",
+                metric_type as "Metric",
+                actual_value as "Actual",
+                prophet_forecast as "Prophet",
+                xgboost_forecast as "XGBoost",
+                pickup_forecast as "Pickup",
+                best_model as "Best Model"
+            FROM actual_vs_forecast
+            WHERE date BETWEEN :from_date - INTERVAL '30 days' AND :from_date
+            ORDER BY date, metric_type
+        """
+        result = await db.execute(text(accuracy_query), {"from_date": from_date, "to_date": to_date})
+        accuracy_df = pd.DataFrame(result.fetchall())
+        if not accuracy_df.empty:
+            accuracy_df.to_excel(writer, sheet_name='Historical Accuracy', index=False)
+
+    output.seek(0)
+
+    filename = f"forecast_{from_date}_{to_date}.xlsx"
+    return StreamingResponse(
+        output,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers={"Content-Disposition": f"attachment; filename={filename}"}
+    )
+
+
+@router.get("/csv/{metric}")
+async def export_csv(
+    metric: str,
+    from_date: Optional[date] = Query(None),
+    to_date: Optional[date] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Download CSV for a specific metric.
+    """
+    if from_date is None:
+        from_date = date.today()
+    if to_date is None:
+        to_date = from_date + timedelta(days=28)
+
+    query = """
+        SELECT
+            f.forecast_date,
+            f.model_type,
+            f.predicted_value,
+            f.lower_bound,
+            f.upper_bound,
+            dm.actual_value,
+            db.budget_value
+        FROM forecasts f
+        LEFT JOIN daily_metrics dm ON f.forecast_date = dm.date AND f.forecast_type = dm.metric_code
+        LEFT JOIN daily_budgets db ON f.forecast_date = db.date AND f.forecast_type = db.budget_type
+        WHERE f.forecast_date BETWEEN :from_date AND :to_date
+            AND f.forecast_type = :metric
+        ORDER BY f.forecast_date, f.model_type
+    """
+
+    result = await db.execute(text(query), {
+        "from_date": from_date,
+        "to_date": to_date,
+        "metric": metric
+    })
+
+    df = pd.DataFrame(result.fetchall())
+
+    output = io.StringIO()
+    df.to_csv(output, index=False)
+    output.seek(0)
+
+    filename = f"{metric}_{from_date}_{to_date}.csv"
+    return StreamingResponse(
+        iter([output.getvalue()]),
+        media_type="text/csv",
+        headers={"Content-Disposition": f"attachment; filename={filename}"}
+    )
+
+
+@router.get("/model-comparison")
+async def export_model_comparison(
+    from_date: Optional[date] = Query(None),
+    to_date: Optional[date] = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Export model comparison data for all metrics.
+    """
+    if from_date is None:
+        from_date = date.today()
+    if to_date is None:
+        to_date = from_date + timedelta(days=28)
+
+    query = """
+        SELECT
+            f.forecast_date,
+            f.forecast_type,
+            fm.metric_name,
+            MAX(CASE WHEN f.model_type = 'prophet' THEN f.predicted_value END) as prophet,
+            MAX(CASE WHEN f.model_type = 'xgboost' THEN f.predicted_value END) as xgboost,
+            MAX(CASE WHEN f.model_type = 'pickup' THEN f.predicted_value END) as pickup,
+            dm.actual_value
+        FROM forecasts f
+        LEFT JOIN forecast_metrics fm ON f.forecast_type = fm.metric_code
+        LEFT JOIN daily_metrics dm ON f.forecast_date = dm.date AND f.forecast_type = dm.metric_code
+        WHERE f.forecast_date BETWEEN :from_date AND :to_date
+        GROUP BY f.forecast_date, f.forecast_type, fm.metric_name, dm.actual_value
+        ORDER BY f.forecast_date, f.forecast_type
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    df = pd.DataFrame(result.fetchall())
+
+    output = io.BytesIO()
+    with pd.ExcelWriter(output, engine='openpyxl') as writer:
+        df.to_excel(writer, sheet_name='Model Comparison', index=False)
+
+    output.seek(0)
+
+    filename = f"model_comparison_{from_date}_{to_date}.xlsx"
+    return StreamingResponse(
+        output,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers={"Content-Disposition": f"attachment; filename={filename}"}
+    )

backend/api/historical.py

@@ -0,0 +1,221 @@
+"""
+Historical data API endpoints
+Provides access to aggregated actual data from daily_occupancy and daily_covers
+"""
+from datetime import date, timedelta
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+@router.get("/occupancy")
+async def get_occupancy_data(
+    from_date: Optional[date] = Query(None, description="Start date"),
+    to_date: Optional[date] = Query(None, description="End date"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get historical occupancy data from daily_occupancy table.
+    """
+    if from_date is None:
+        from_date = date.today() - timedelta(days=30)
+    if to_date is None:
+        to_date = date.today()
+
+    query = """
+        SELECT
+            date,
+            total_rooms,
+            occupied_rooms,
+            occupancy_pct,
+            total_guests,
+            total_adults,
+            total_children,
+            total_infants,
+            arrival_count,
+            room_revenue,
+            adr,
+            revpar,
+            agr,
+            breakfast_allocation_qty,
+            dinner_allocation_qty,
+            by_room_type,
+            revenue_by_room_type
+        FROM daily_occupancy
+        WHERE date BETWEEN :from_date AND :to_date
+        ORDER BY date
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.date,
+            "total_rooms": row.total_rooms,
+            "occupied_rooms": row.occupied_rooms,
+            "occupancy_pct": float(row.occupancy_pct) if row.occupancy_pct else 0,
+            "total_guests": row.total_guests,
+            "total_adults": row.total_adults,
+            "total_children": row.total_children,
+            "total_infants": row.total_infants,
+            "arrival_count": row.arrival_count,
+            "room_revenue": float(row.room_revenue) if row.room_revenue else 0,
+            "adr": float(row.adr) if row.adr else 0,
+            "revpar": float(row.revpar) if row.revpar else 0,
+            "agr": float(row.agr) if row.agr else 0,
+            "breakfast_allocation_qty": row.breakfast_allocation_qty,
+            "dinner_allocation_qty": row.dinner_allocation_qty,
+            "by_room_type": row.by_room_type,
+            "revenue_by_room_type": row.revenue_by_room_type
+        }
+        for row in rows
+    ]
+
+
+@router.get("/covers")
+async def get_covers_data(
+    from_date: Optional[date] = Query(None, description="Start date"),
+    to_date: Optional[date] = Query(None, description="End date"),
+    service_period: Optional[str] = Query(None, description="Filter by: lunch, dinner"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get historical covers data from daily_covers table.
+    """
+    if from_date is None:
+        from_date = date.today() - timedelta(days=30)
+    if to_date is None:
+        to_date = date.today()
+
+    query = """
+        SELECT
+            date,
+            service_period,
+            total_bookings,
+            total_covers,
+            avg_party_size,
+            hotel_guest_covers,
+            external_covers,
+            dbb_covers,
+            package_covers,
+            cancelled_bookings,
+            cancelled_covers,
+            no_show_bookings,
+            no_show_covers,
+            by_source
+        FROM daily_covers
+        WHERE date BETWEEN :from_date AND :to_date
+    """
+
+    params = {"from_date": from_date, "to_date": to_date}
+
+    if service_period:
+        query += " AND service_period = :service_period"
+        params["service_period"] = service_period
+
+    query += " ORDER BY date, service_period"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "date": row.date,
+            "service_period": row.service_period,
+            "total_bookings": row.total_bookings,
+            "total_covers": row.total_covers,
+            "avg_party_size": float(row.avg_party_size) if row.avg_party_size else 0,
+            "hotel_guest_covers": row.hotel_guest_covers,
+            "external_covers": row.external_covers,
+            "dbb_covers": row.dbb_covers,
+            "package_covers": row.package_covers,
+            "cancelled_bookings": row.cancelled_bookings,
+            "cancelled_covers": row.cancelled_covers,
+            "no_show_bookings": row.no_show_bookings,
+            "no_show_covers": row.no_show_covers,
+            "by_source": row.by_source
+        }
+        for row in rows
+    ]
+
+
+@router.get("/summary")
+async def get_daily_summary(
+    from_date: Optional[date] = Query(None, description="Start date"),
+    to_date: Optional[date] = Query(None, description="End date"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get combined daily summary with occupancy and covers data.
+    """
+    if from_date is None:
+        from_date = date.today() - timedelta(days=30)
+    if to_date is None:
+        to_date = date.today()
+
+    query = """
+        SELECT
+            o.date,
+            EXTRACT(DOW FROM o.date) as day_of_week,
+            o.total_rooms,
+            o.available_rooms,
+            o.occupied_rooms,
+            o.occupancy_pct,
+            o.total_guests,
+            o.room_revenue,
+            o.adr,
+            o.revpar,
+            o.agr,
+            o.arrival_count,
+            o.breakfast_allocation_qty,
+            o.dinner_allocation_qty,
+            COALESCE(cl.total_covers, 0) as lunch_covers,
+            COALESCE(cd.total_covers, 0) as dinner_covers,
+            COALESCE(cl.total_bookings, 0) as lunch_bookings,
+            COALESCE(cd.total_bookings, 0) as dinner_bookings
+        FROM daily_occupancy o
+        LEFT JOIN daily_covers cl ON o.date = cl.date AND cl.service_period = 'lunch'
+        LEFT JOIN daily_covers cd ON o.date = cd.date AND cd.service_period = 'dinner'
+        WHERE o.date BETWEEN :from_date AND :to_date
+        ORDER BY o.date
+    """
+
+    result = await db.execute(text(query), {"from_date": from_date, "to_date": to_date})
+    rows = result.fetchall()
+
+    day_names = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
+
+    return [
+        {
+            "date": row.date,
+            "day_of_week": day_names[int(row.day_of_week)],
+            "total_rooms": row.total_rooms,
+            "available_rooms": row.available_rooms,
+            "occupied_rooms": row.occupied_rooms,
+            "occupancy_pct": float(row.occupancy_pct) if row.occupancy_pct else 0,
+            "total_guests": row.total_guests,
+            "room_revenue": float(row.room_revenue) if row.room_revenue else 0,
+            "adr": float(row.adr) if row.adr else 0,
+            "revpar": float(row.revpar) if row.revpar else 0,
+            "agr": float(row.agr) if row.agr else 0,
+            "arrival_count": row.arrival_count,
+            "breakfast_allocation_qty": row.breakfast_allocation_qty,
+            "dinner_allocation_qty": row.dinner_allocation_qty,
+            "lunch_covers": row.lunch_covers,
+            "dinner_covers": row.dinner_covers,
+            "lunch_bookings": row.lunch_bookings,
+            "dinner_bookings": row.dinner_bookings
+        }
+        for row in rows
+    ]

backend/api/public.py

@@ -0,0 +1,537 @@
+"""
+Public API endpoints - accessible with API key authentication
+For external integrations like Kitchen Flash app
+"""
+from datetime import date, datetime, timedelta
+from typing import Optional
+from fastapi import APIRouter, Depends, Query, HTTPException
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+import logging
+
+from database import get_db
+from auth import get_api_key_auth
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+def get_prior_year_date(target_date: date) -> date:
+    """
+    Get prior year date with 364-day offset for day-of-week alignment.
+    52 weeks = 364 days, so Monday aligns with Monday.
+    """
+    return target_date - timedelta(days=364)
+
+
+@router.get("/forecast/rooms")
+async def get_rooms_forecast(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    days: int = Query(7, ge=1, le=365, description="Number of days to forecast"),
+    db: AsyncSession = Depends(get_db),
+    api_key: dict = Depends(get_api_key_auth)
+):
+    """
+    Get forecasted room bookings for external applications.
+
+    Returns: OTB rooms, forecast rooms, occupancy %, prior year data
+    """
+    from services.forecasting.pickup_v2_model import forecast_rooms_for_date
+
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    end = start + timedelta(days=days - 1)
+    today = date.today()
+
+    # Get total rooms for occupancy calculation
+    result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'total_rooms'")
+    )
+    row = result.fetchone()
+    total_rooms = int(row.config_value) if row and row.config_value else 30
+
+    data = []
+    current = start
+    while current <= end:
+        lead_days = (current - today).days if current >= today else 0
+        prior_date = get_prior_year_date(current)
+
+        try:
+            if current >= today:
+                # Future date - get forecast
+                forecast = await forecast_rooms_for_date(
+                    db, current, lead_days, prior_date, 'hotel_room_nights'
+                )
+                otb_rooms = forecast.get('current_otb', 0) or 0
+                pickup_rooms = forecast.get('expected_pickup', 0) or 0
+                forecast_rooms = otb_rooms + pickup_rooms
+                prior_otb = forecast.get('prior_year_otb', 0) or 0
+                prior_final = forecast.get('prior_year_final', 0) or 0
+
+                # Get guest counts from stats
+                stats_result = await db.execute(
+                    text("""
+                        SELECT booking_count, guests_count FROM newbook_bookings_stats
+                        WHERE date = :target_date
+                    """),
+                    {"target_date": current}
+                )
+                stats_row = stats_result.fetchone()
+                otb_guests = stats_row.guests_count if stats_row and stats_row.guests_count else 0
+
+                # Prior year guests for ratio
+                prior_stats = await db.execute(
+                    text("""
+                        SELECT booking_count, guests_count FROM newbook_bookings_stats
+                        WHERE date = :prior_date
+                    """),
+                    {"prior_date": prior_date}
+                )
+                prior_stats_row = prior_stats.fetchone()
+                prior_guests = prior_stats_row.guests_count if prior_stats_row and prior_stats_row.guests_count else 0
+                prior_rooms_actual = prior_stats_row.booking_count if prior_stats_row and prior_stats_row.booking_count else 0
+
+                # Calculate guests per room ratio (prior year -> current OTB -> default)
+                if prior_rooms_actual > 0:
+                    guests_per_room = prior_guests / prior_rooms_actual
+                elif otb_rooms > 0:
+                    guests_per_room = otb_guests / otb_rooms
+                else:
+                    guests_per_room = 1.8
+
+                pickup_guests = round(pickup_rooms * guests_per_room)
+                forecast_guests = otb_guests + pickup_guests
+            else:
+                # Past date - get actuals from stats
+                stats_result = await db.execute(
+                    text("""
+                        SELECT booking_count, guests_count FROM newbook_bookings_stats
+                        WHERE date = :target_date
+                    """),
+                    {"target_date": current}
+                )
+                stats_row = stats_result.fetchone()
+                otb_rooms = stats_row.booking_count if stats_row else 0
+                otb_guests = stats_row.guests_count if stats_row and stats_row.guests_count else 0
+                forecast_rooms = otb_rooms
+                forecast_guests = otb_guests
+                pickup_rooms = 0
+                pickup_guests = 0
+
+                # Prior year stats
+                prior_result = await db.execute(
+                    text("""
+                        SELECT booking_count FROM newbook_bookings_stats
+                        WHERE date = :prior_date
+                    """),
+                    {"prior_date": prior_date}
+                )
+                prior_row = prior_result.fetchone()
+                prior_final = prior_row.booking_count if prior_row else 0
+                prior_otb = prior_final
+
+            occupancy_pct = round((forecast_rooms / total_rooms) * 100, 1) if total_rooms > 0 else 0
+
+            data.append({
+                "date": current.isoformat(),
+                "day": current.strftime("%A"),
+                "lead_days": lead_days,
+                "otb_rooms": otb_rooms,
+                "pickup_rooms": pickup_rooms,
+                "forecast_rooms": forecast_rooms,
+                "otb_guests": otb_guests,
+                "pickup_guests": pickup_guests,
+                "forecast_guests": forecast_guests,
+                "occupancy_pct": occupancy_pct,
+                "prior_year_otb": prior_otb,
+                "prior_year_final": prior_final,
+            })
+        except Exception as e:
+            logger.warning(f"Error forecasting rooms for {current}: {e}")
+            data.append({
+                "date": current.isoformat(),
+                "day": current.strftime("%A"),
+                "lead_days": lead_days,
+                "otb_rooms": 0,
+                "pickup_rooms": 0,
+                "forecast_rooms": 0,
+                "otb_guests": 0,
+                "pickup_guests": 0,
+                "forecast_guests": 0,
+                "occupancy_pct": 0,
+                "prior_year_otb": 0,
+                "prior_year_final": 0,
+                "error": str(e)
+            })
+
+        current += timedelta(days=1)
+
+    return {"data": data, "total_rooms": total_rooms}
+
+
+@router.get("/forecast/covers")
+async def get_covers_forecast(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    days: int = Query(7, ge=1, le=365, description="Number of days to forecast"),
+    db: AsyncSession = Depends(get_db),
+    api_key: dict = Depends(get_api_key_auth)
+):
+    """
+    Get forecasted restaurant covers by period for external applications.
+
+    Returns: OTB covers, forecast covers, prior year data for breakfast, lunch, dinner
+    """
+    from services.forecasting.covers_model import forecast_covers_range
+
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    end = start + timedelta(days=days - 1)
+    today = date.today()
+
+    try:
+        # Get covers forecast
+        covers_data = await forecast_covers_range(db, start, end, include_details=False)
+    except Exception as e:
+        logger.error(f"Covers forecast failed: {e}")
+        raise HTTPException(status_code=500, detail=f"Forecast error: {str(e)}")
+
+    # Get prior year covers from stats
+    prior_start = get_prior_year_date(start)
+    prior_end = get_prior_year_date(end)
+
+    prior_result = await db.execute(
+        text("""
+            SELECT date, breakfast_covers, lunch_covers, dinner_covers
+            FROM resos_bookings_stats
+            WHERE date >= :start_date AND date <= :end_date
+        """),
+        {"start_date": prior_start, "end_date": prior_end}
+    )
+    prior_rows = prior_result.fetchall()
+    prior_by_date = {row.date: row for row in prior_rows}
+
+    data = []
+    for day_data in covers_data.get("data", []):
+        target_date = datetime.strptime(day_data["date"], "%Y-%m-%d").date()
+        prior_date = get_prior_year_date(target_date)
+        lead_days = (target_date - today).days if target_date >= today else 0
+
+        prior_row = prior_by_date.get(prior_date)
+        prior_breakfast = prior_row.breakfast_covers if prior_row else 0
+        prior_lunch = prior_row.lunch_covers if prior_row else 0
+        prior_dinner = prior_row.dinner_covers if prior_row else 0
+
+        # For prior OTB, we don't have pace data, so use same as final for simplicity
+        data.append({
+            "date": day_data["date"],
+            "day": day_data["day_of_week"],
+            "lead_days": lead_days,
+            "breakfast": {
+                "otb": day_data["breakfast"]["otb"],
+                "forecast": day_data["breakfast"]["forecast"],
+                "prior_otb": prior_breakfast,  # Same as final for prior year
+                "prior_final": prior_breakfast
+            },
+            "lunch": {
+                "otb": day_data["lunch"]["otb"],
+                "forecast": day_data["lunch"]["forecast"],
+                "prior_otb": prior_lunch,
+                "prior_final": prior_lunch
+            },
+            "dinner": {
+                "otb": day_data["dinner"]["otb"],
+                "forecast": day_data["dinner"]["forecast"],
+                "prior_otb": prior_dinner,
+                "prior_final": prior_dinner
+            },
+            "total": {
+                "otb": day_data["totals"]["otb"],
+                "forecast": day_data["totals"]["forecast"],
+                "prior_final": prior_breakfast + prior_lunch + prior_dinner
+            }
+        })
+
+    return {"data": data}
+
+
+@router.get("/forecast/revenue")
+async def get_revenue_forecast(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    days: int = Query(7, ge=1, le=365, description="Number of days to forecast"),
+    type: str = Query("all", description="Revenue type: all, accom, dry, wet"),
+    db: AsyncSession = Depends(get_db),
+    api_key: dict = Depends(get_api_key_auth)
+):
+    """
+    Get forecasted revenue by type for external applications.
+
+    Returns: OTB revenue, forecast revenue, prior year actuals, budget
+    """
+    from services.forecasting.covers_model import forecast_covers_range
+    from services.forecasting.pickup_v2_model import forecast_revenue_for_date
+
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    end = start + timedelta(days=days - 1)
+    today = date.today()
+    VAT_RATE = 1.20
+
+    # Get spend settings for restaurant revenue
+    spend_result = await db.execute(
+        text("""
+            SELECT config_key, config_value
+            FROM system_config
+            WHERE config_key LIKE 'resos_%_spend'
+        """)
+    )
+    spend_rows = spend_result.fetchall()
+    spend_settings = {row.config_key: float(row.config_value or 0) for row in spend_rows}
+
+    def get_spend_by_period(period: str, revenue_type: str) -> float:
+        if revenue_type == 'dry':
+            return spend_settings.get(f'resos_{period}_food_spend', 0) / VAT_RATE
+        elif revenue_type == 'wet':
+            return spend_settings.get(f'resos_{period}_drinks_spend', 0) / VAT_RATE
+        else:
+            food = spend_settings.get(f'resos_{period}_food_spend', 0)
+            drinks = spend_settings.get(f'resos_{period}_drinks_spend', 0)
+            return (food + drinks) / VAT_RATE
+
+    # Get actual revenue
+    actual_result = await db.execute(
+        text("""
+            SELECT date, accommodation, dry, wet
+            FROM newbook_net_revenue_data
+            WHERE date >= :start_date AND date <= :end_date
+        """),
+        {"start_date": start, "end_date": end}
+    )
+    actual_by_date = {row.date: row for row in actual_result.fetchall()}
+
+    # Get prior year revenue
+    prior_start = get_prior_year_date(start)
+    prior_end = get_prior_year_date(end)
+    prior_result = await db.execute(
+        text("""
+            SELECT date, accommodation, dry, wet
+            FROM newbook_net_revenue_data
+            WHERE date >= :start_date AND date <= :end_date
+        """),
+        {"start_date": prior_start, "end_date": prior_end}
+    )
+    prior_by_date = {row.date: row for row in prior_result.fetchall()}
+
+    # Get budgets
+    budget_types = []
+    if type in ['all', 'accom']:
+        budget_types.append('net_accom')
+    if type in ['all', 'dry']:
+        budget_types.append('net_dry')
+    if type in ['all', 'wet']:
+        budget_types.append('net_wet')
+    if type == 'all':
+        budget_types.append('total_rev')
+
+    budget_result = await db.execute(
+        text("""
+            SELECT date, budget_type, budget_value
+            FROM daily_budgets
+            WHERE date >= :start_date AND date <= :end_date
+            AND budget_type = ANY(:types)
+        """),
+        {"start_date": start, "end_date": end, "types": budget_types if budget_types else ['']}
+    )
+    budget_data = {}
+    for row in budget_result.fetchall():
+        if row.date not in budget_data:
+            budget_data[row.date] = {}
+        budget_data[row.date][row.budget_type] = float(row.budget_value or 0)
+
+    # Get covers forecast for restaurant revenue
+    try:
+        covers_data = await forecast_covers_range(db, start, end, include_details=False)
+        covers_by_date = {c["date"]: c for c in covers_data.get("data", [])}
+    except Exception as e:
+        logger.warning(f"Covers forecast failed: {e}")
+        covers_by_date = {}
+
+    data = []
+    current = start
+    while current <= end:
+        is_past = current < today
+        lead_days = (current - today).days if current >= today else 0
+        prior_date = get_prior_year_date(current)
+
+        # Prior year actual
+        prior_row = prior_by_date.get(prior_date)
+        prior_accom = float(prior_row.accommodation) if prior_row and prior_row.accommodation else 0
+        prior_dry = float(prior_row.dry) if prior_row and prior_row.dry else 0
+        prior_wet = float(prior_row.wet) if prior_row and prior_row.wet else 0
+
+        # Budget
+        day_budget = budget_data.get(current, {})
+        budget_accom = day_budget.get('net_accom', 0)
+        budget_dry = day_budget.get('net_dry', 0)
+        budget_wet = day_budget.get('net_wet', 0)
+        budget_total = day_budget.get('total_rev', budget_accom + budget_dry + budget_wet)
+
+        if is_past:
+            # Past: use actual revenue
+            actual_row = actual_by_date.get(current)
+            accom_otb = float(actual_row.accommodation) if actual_row and actual_row.accommodation else 0
+            dry_otb = float(actual_row.dry) if actual_row and actual_row.dry else 0
+            wet_otb = float(actual_row.wet) if actual_row and actual_row.wet else 0
+            accom_forecast = accom_otb
+            dry_forecast = dry_otb
+            wet_forecast = wet_otb
+            accom_prior_otb = prior_accom
+        else:
+            # Future: forecast
+            # Accommodation - use revenue forecast model
+            try:
+                accom_forecast_data = await forecast_revenue_for_date(
+                    db, current, lead_days, prior_date
+                )
+                accom_otb = accom_forecast_data.get('current_otb_rev', 0) or 0
+                accom_pickup = accom_forecast_data.get('forecast_pickup_rev', 0) or 0
+                accom_forecast = accom_otb + accom_pickup
+                accom_prior_otb = accom_forecast_data.get('prior_year_otb_rev', 0) or 0
+            except Exception:
+                accom_otb = accom_forecast = accom_prior_otb = 0
+
+            # Restaurant
+            day_covers = covers_by_date.get(current.isoformat())
+            if day_covers:
+                breakfast_otb = day_covers["breakfast"]["otb"]
+                lunch_otb = day_covers["lunch"]["otb"]
+                dinner_otb = day_covers["dinner"]["otb"]
+                breakfast_forecast = day_covers["breakfast"]["forecast"]
+                lunch_forecast = day_covers["lunch"]["forecast"]
+                dinner_forecast = day_covers["dinner"]["forecast"]
+
+                dry_otb = (
+                    breakfast_otb * get_spend_by_period('breakfast', 'dry') +
+                    lunch_otb * get_spend_by_period('lunch', 'dry') +
+                    dinner_otb * get_spend_by_period('dinner', 'dry')
+                )
+                dry_forecast = (
+                    breakfast_forecast * get_spend_by_period('breakfast', 'dry') +
+                    lunch_forecast * get_spend_by_period('lunch', 'dry') +
+                    dinner_forecast * get_spend_by_period('dinner', 'dry')
+                )
+                wet_otb = (
+                    breakfast_otb * get_spend_by_period('breakfast', 'wet') +
+                    lunch_otb * get_spend_by_period('lunch', 'wet') +
+                    dinner_otb * get_spend_by_period('dinner', 'wet')
+                )
+                wet_forecast = (
+                    breakfast_forecast * get_spend_by_period('breakfast', 'wet') +
+                    lunch_forecast * get_spend_by_period('lunch', 'wet') +
+                    dinner_forecast * get_spend_by_period('dinner', 'wet')
+                )
+            else:
+                dry_otb = dry_forecast = wet_otb = wet_forecast = 0
+
+        day_data = {
+            "date": current.isoformat(),
+            "day": current.strftime("%A"),
+            "is_past": is_past,
+            "lead_days": lead_days,
+        }
+
+        if type in ['all', 'accom']:
+            day_data["accom"] = {
+                "otb": round(accom_otb, 2),
+                "forecast": round(accom_forecast, 2),
+                "prior_otb": round(accom_prior_otb, 2) if not is_past else round(prior_accom, 2),
+                "prior_final": round(prior_accom, 2),
+                "budget": round(budget_accom, 2)
+            }
+
+        if type in ['all', 'dry']:
+            day_data["dry"] = {
+                "otb": round(dry_otb, 2),
+                "forecast": round(dry_forecast, 2),
+                "prior_final": round(prior_dry, 2),
+                "budget": round(budget_dry, 2)
+            }
+
+        if type in ['all', 'wet']:
+            day_data["wet"] = {
+                "otb": round(wet_otb, 2),
+                "forecast": round(wet_forecast, 2),
+                "prior_final": round(prior_wet, 2),
+                "budget": round(budget_wet, 2)
+            }
+
+        if type == 'all':
+            total_otb = accom_otb + dry_otb + wet_otb
+            total_forecast = accom_forecast + dry_forecast + wet_forecast
+            total_prior = prior_accom + prior_dry + prior_wet
+            day_data["total"] = {
+                "otb": round(total_otb, 2),
+                "forecast": round(total_forecast, 2),
+                "prior_final": round(total_prior, 2),
+                "budget": round(budget_total, 2)
+            }
+
+        data.append(day_data)
+        current += timedelta(days=1)
+
+    return {"data": data}
+
+
+@router.get("/forecast/spend-rates")
+async def get_spend_rates(
+    db: AsyncSession = Depends(get_db),
+    api_key: dict = Depends(get_api_key_auth)
+):
+    """
+    Return spend-per-cover rates for each meal period.
+    Values are gross (inc VAT) from system_config, plus the VAT rate.
+    Kitchen app can divide by VAT rate to get net values.
+    """
+    VAT_RATE = 1.20
+
+    spend_result = await db.execute(
+        text("""
+            SELECT config_key, config_value
+            FROM system_config
+            WHERE config_key LIKE 'resos_%_spend'
+        """)
+    )
+    spend_rows = spend_result.fetchall()
+    spend_settings = {row.config_key: float(row.config_value or 0) for row in spend_rows}
+
+    return {
+        "vat_rate": VAT_RATE,
+        "periods": {
+            "breakfast": {
+                "food_spend_gross": spend_settings.get("resos_breakfast_food_spend", 0),
+                "drinks_spend_gross": spend_settings.get("resos_breakfast_drinks_spend", 0),
+                "food_spend_net": round(spend_settings.get("resos_breakfast_food_spend", 0) / VAT_RATE, 2),
+                "drinks_spend_net": round(spend_settings.get("resos_breakfast_drinks_spend", 0) / VAT_RATE, 2),
+            },
+            "lunch": {
+                "food_spend_gross": spend_settings.get("resos_lunch_food_spend", 0),
+                "drinks_spend_gross": spend_settings.get("resos_lunch_drinks_spend", 0),
+                "food_spend_net": round(spend_settings.get("resos_lunch_food_spend", 0) / VAT_RATE, 2),
+                "drinks_spend_net": round(spend_settings.get("resos_lunch_drinks_spend", 0) / VAT_RATE, 2),
+            },
+            "dinner": {
+                "food_spend_gross": spend_settings.get("resos_dinner_food_spend", 0),
+                "drinks_spend_gross": spend_settings.get("resos_dinner_drinks_spend", 0),
+                "food_spend_net": round(spend_settings.get("resos_dinner_food_spend", 0) / VAT_RATE, 2),
+                "drinks_spend_net": round(spend_settings.get("resos_dinner_drinks_spend", 0) / VAT_RATE, 2),
+            },
+        }
+    }

backend/api/reports.py

@@ -0,0 +1,927 @@
+"""
+Reports API endpoints
+Provide aggregated data for frontend reports and visualizations
+"""
+from datetime import date, datetime, timedelta
+from typing import Optional, List, Dict, Any
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+import calendar
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+# ============================================
+# RESPONSE MODELS
+# ============================================
+
+class OccupancyDataPoint(BaseModel):
+    date: str
+    total_occupancy_pct: Optional[float] = None
+    bookable_occupancy_pct: Optional[float] = None
+    booking_count: int = 0
+    rooms_count: int = 0
+    bookable_count: int = 0
+
+
+class BookingsDataPoint(BaseModel):
+    date: str
+    booking_count: int = 0
+    guests_count: int = 0
+    rooms_count: int = 0
+
+
+class RatesDataPoint(BaseModel):
+    date: str
+    guest_rate_total: float = 0.0
+    net_booking_rev_total: float = 0.0
+    booking_count: int = 0
+    avg_guest_rate: Optional[float] = None
+    avg_net_rate: Optional[float] = None
+
+
+class RevenueDataPoint(BaseModel):
+    date: str
+    accommodation: float = 0.0
+    dry: float = 0.0
+    wet: float = 0.0
+    total: float = 0.0
+
+
+# ============================================
+# OCCUPANCY REPORT ENDPOINT
+# ============================================
+
+@router.get("/occupancy", response_model=List[OccupancyDataPoint])
+async def get_occupancy_report(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    consolidation: str = Query("day", description="Consolidation period: day, week, or month"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get occupancy data for the specified date range with optional consolidation.
+
+    - **day**: Returns daily data points
+    - **week**: Aggregates by week (Monday start)
+    - **month**: Aggregates by month
+    """
+    # Validate dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    if consolidation not in ["day", "week", "month"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day', 'week', or 'month'")
+
+    # Build the query based on consolidation type
+    # Use generate_series to include all dates in range, even those without data
+    if consolidation == "day":
+        query = text("""
+            SELECT
+                to_char(d, 'YYYY-MM-DD') as period_date,
+                COALESCE(s.total_occupancy_pct, 0) as total_occupancy_pct,
+                COALESCE(s.bookable_occupancy_pct, 0) as bookable_occupancy_pct,
+                COALESCE(s.booking_count, 0) as booking_count,
+                COALESCE(s.rooms_count, 0) as rooms_count,
+                COALESCE(s.bookable_count, 0) as bookable_count
+            FROM generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) d
+            LEFT JOIN newbook_bookings_stats s ON s.date = d
+            ORDER BY d
+        """)
+    elif consolidation == "week":
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            weekly_data AS (
+                SELECT
+                    CAST(date_trunc('week', d) AS date) as week_start,
+                    ROUND(CAST(AVG(COALESCE(s.total_occupancy_pct, 0)) AS numeric), 2) as total_occupancy_pct,
+                    ROUND(CAST(AVG(COALESCE(s.bookable_occupancy_pct, 0)) AS numeric), 2) as bookable_occupancy_pct,
+                    CAST(SUM(COALESCE(s.booking_count, 0)) AS integer) as booking_count,
+                    CAST(ROUND(AVG(COALESCE(s.rooms_count, 0))) AS integer) as rooms_count,
+                    CAST(ROUND(AVG(COALESCE(s.bookable_count, 0))) AS integer) as bookable_count
+                FROM date_range dr
+                LEFT JOIN newbook_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('week', d)
+            )
+            SELECT
+                to_char(week_start, 'YYYY-MM-DD') as period_date,
+                total_occupancy_pct,
+                bookable_occupancy_pct,
+                booking_count,
+                rooms_count,
+                bookable_count
+            FROM weekly_data
+            ORDER BY week_start
+        """)
+    else:  # month
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            monthly_data AS (
+                SELECT
+                    CAST(date_trunc('month', d) AS date) as month_start,
+                    ROUND(CAST(AVG(COALESCE(s.total_occupancy_pct, 0)) AS numeric), 2) as total_occupancy_pct,
+                    ROUND(CAST(AVG(COALESCE(s.bookable_occupancy_pct, 0)) AS numeric), 2) as bookable_occupancy_pct,
+                    CAST(SUM(COALESCE(s.booking_count, 0)) AS integer) as booking_count,
+                    CAST(ROUND(AVG(COALESCE(s.rooms_count, 0))) AS integer) as rooms_count,
+                    CAST(ROUND(AVG(COALESCE(s.bookable_count, 0))) AS integer) as bookable_count
+                FROM date_range dr
+                LEFT JOIN newbook_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('month', d)
+            )
+            SELECT
+                to_char(month_start, 'YYYY-MM-DD') as period_date,
+                total_occupancy_pct,
+                bookable_occupancy_pct,
+                booking_count,
+                rooms_count,
+                bookable_count
+            FROM monthly_data
+            ORDER BY month_start
+        """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    data_points = []
+    for row in rows:
+        data_points.append(OccupancyDataPoint(
+            date=row.period_date,
+            total_occupancy_pct=float(row.total_occupancy_pct) if row.total_occupancy_pct else None,
+            bookable_occupancy_pct=float(row.bookable_occupancy_pct) if row.bookable_occupancy_pct else None,
+            booking_count=row.booking_count or 0,
+            rooms_count=row.rooms_count or 0,
+            bookable_count=row.bookable_count or 0,
+        ))
+
+    return data_points
+
+
+# ============================================
+# BOOKINGS REPORT ENDPOINT
+# ============================================
+
+@router.get("/bookings", response_model=List[BookingsDataPoint])
+async def get_bookings_report(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    consolidation: str = Query("day", description="Consolidation period: day, week, or month"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get bookings and guests data for the specified date range with optional consolidation.
+    """
+    # Validate dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    if consolidation not in ["day", "week", "month"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day', 'week', or 'month'")
+
+    if consolidation == "day":
+        query = text("""
+            SELECT
+                to_char(d, 'YYYY-MM-DD') as period_date,
+                COALESCE(s.booking_count, 0) as booking_count,
+                COALESCE(s.guests_count, 0) as guests_count,
+                COALESCE(s.rooms_count, 0) as rooms_count
+            FROM generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) d
+            LEFT JOIN newbook_bookings_stats s ON s.date = d
+            ORDER BY d
+        """)
+    elif consolidation == "week":
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            weekly_data AS (
+                SELECT
+                    CAST(date_trunc('week', d) AS date) as week_start,
+                    CAST(SUM(COALESCE(s.booking_count, 0)) AS integer) as booking_count,
+                    CAST(SUM(COALESCE(s.guests_count, 0)) AS integer) as guests_count,
+                    CAST(ROUND(AVG(COALESCE(s.rooms_count, 0))) AS integer) as rooms_count
+                FROM date_range dr
+                LEFT JOIN newbook_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('week', d)
+            )
+            SELECT
+                to_char(week_start, 'YYYY-MM-DD') as period_date,
+                booking_count,
+                guests_count,
+                rooms_count
+            FROM weekly_data
+            ORDER BY week_start
+        """)
+    else:  # month
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            monthly_data AS (
+                SELECT
+                    CAST(date_trunc('month', d) AS date) as month_start,
+                    CAST(SUM(COALESCE(s.booking_count, 0)) AS integer) as booking_count,
+                    CAST(SUM(COALESCE(s.guests_count, 0)) AS integer) as guests_count,
+                    CAST(ROUND(AVG(COALESCE(s.rooms_count, 0))) AS integer) as rooms_count
+                FROM date_range dr
+                LEFT JOIN newbook_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('month', d)
+            )
+            SELECT
+                to_char(month_start, 'YYYY-MM-DD') as period_date,
+                booking_count,
+                guests_count,
+                rooms_count
+            FROM monthly_data
+            ORDER BY month_start
+        """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    data_points = []
+    for row in rows:
+        data_points.append(BookingsDataPoint(
+            date=row.period_date,
+            booking_count=row.booking_count or 0,
+            guests_count=row.guests_count or 0,
+            rooms_count=row.rooms_count or 0,
+        ))
+
+    return data_points
+
+
+# ============================================
+# RATES REPORT ENDPOINT
+# ============================================
+
+@router.get("/rates", response_model=List[RatesDataPoint])
+async def get_rates_report(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    consolidation: str = Query("day", description="Consolidation period: day, week, or month"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get guest rates data (gross tariff / calculated amount) for the specified date range.
+    """
+    # Validate dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    if consolidation not in ["day", "week", "month"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day', 'week', or 'month'")
+
+    if consolidation == "day":
+        query = text("""
+            SELECT
+                to_char(d, 'YYYY-MM-DD') as period_date,
+                COALESCE(s.guest_rate_total, 0) as guest_rate_total,
+                COALESCE(s.net_booking_rev_total, 0) as net_booking_rev_total,
+                COALESCE(s.booking_count, 0) as booking_count
+            FROM generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) d
+            LEFT JOIN newbook_bookings_stats s ON s.date = d
+            ORDER BY d
+        """)
+    elif consolidation == "week":
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            weekly_data AS (
+                SELECT
+                    CAST(date_trunc('week', d) AS date) as week_start,
+                    ROUND(CAST(SUM(COALESCE(s.guest_rate_total, 0)) AS numeric), 2) as guest_rate_total,
+                    ROUND(CAST(SUM(COALESCE(s.net_booking_rev_total, 0)) AS numeric), 2) as net_booking_rev_total,
+                    CAST(SUM(COALESCE(s.booking_count, 0)) AS integer) as booking_count
+                FROM date_range dr
+                LEFT JOIN newbook_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('week', d)
+            )
+            SELECT
+                to_char(week_start, 'YYYY-MM-DD') as period_date,
+                guest_rate_total,
+                net_booking_rev_total,
+                booking_count
+            FROM weekly_data
+            ORDER BY week_start
+        """)
+    else:  # month
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            monthly_data AS (
+                SELECT
+                    CAST(date_trunc('month', d) AS date) as month_start,
+                    ROUND(CAST(SUM(COALESCE(s.guest_rate_total, 0)) AS numeric), 2) as guest_rate_total,
+                    ROUND(CAST(SUM(COALESCE(s.net_booking_rev_total, 0)) AS numeric), 2) as net_booking_rev_total,
+                    CAST(SUM(COALESCE(s.booking_count, 0)) AS integer) as booking_count
+                FROM date_range dr
+                LEFT JOIN newbook_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('month', d)
+            )
+            SELECT
+                to_char(month_start, 'YYYY-MM-DD') as period_date,
+                guest_rate_total,
+                net_booking_rev_total,
+                booking_count
+            FROM monthly_data
+            ORDER BY month_start
+        """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    data_points = []
+    for row in rows:
+        guest_rate = float(row.guest_rate_total) if row.guest_rate_total else 0.0
+        net_rate = float(row.net_booking_rev_total) if row.net_booking_rev_total else 0.0
+        bookings = row.booking_count or 0
+
+        data_points.append(RatesDataPoint(
+            date=row.period_date,
+            guest_rate_total=guest_rate,
+            net_booking_rev_total=net_rate,
+            booking_count=bookings,
+            avg_guest_rate=round(guest_rate / bookings, 2) if bookings > 0 else None,
+            avg_net_rate=round(net_rate / bookings, 2) if bookings > 0 else None,
+        ))
+
+    return data_points
+
+
+# ============================================
+# REVENUE REPORT ENDPOINT
+# ============================================
+
+@router.get("/revenue", response_model=List[RevenueDataPoint])
+async def get_revenue_report(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    consolidation: str = Query("day", description="Consolidation period: day, week, or month"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get net revenue data (accommodation, dry, wet) for the specified date range.
+    """
+    # Validate dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    if consolidation not in ["day", "week", "month"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day', 'week', or 'month'")
+
+    if consolidation == "day":
+        query = text("""
+            SELECT
+                to_char(d, 'YYYY-MM-DD') as period_date,
+                COALESCE(r.accommodation, 0) as accommodation,
+                COALESCE(r.dry, 0) as dry,
+                COALESCE(r.wet, 0) as wet
+            FROM generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) d
+            LEFT JOIN newbook_net_revenue_data r ON r.date = d
+            ORDER BY d
+        """)
+    elif consolidation == "week":
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            weekly_data AS (
+                SELECT
+                    CAST(date_trunc('week', d) AS date) as week_start,
+                    ROUND(CAST(SUM(COALESCE(r.accommodation, 0)) AS numeric), 2) as accommodation,
+                    ROUND(CAST(SUM(COALESCE(r.dry, 0)) AS numeric), 2) as dry,
+                    ROUND(CAST(SUM(COALESCE(r.wet, 0)) AS numeric), 2) as wet
+                FROM date_range dr
+                LEFT JOIN newbook_net_revenue_data r ON r.date = dr.d
+                GROUP BY date_trunc('week', d)
+            )
+            SELECT
+                to_char(week_start, 'YYYY-MM-DD') as period_date,
+                accommodation,
+                dry,
+                wet
+            FROM weekly_data
+            ORDER BY week_start
+        """)
+    else:  # month
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            monthly_data AS (
+                SELECT
+                    CAST(date_trunc('month', d) AS date) as month_start,
+                    ROUND(CAST(SUM(COALESCE(r.accommodation, 0)) AS numeric), 2) as accommodation,
+                    ROUND(CAST(SUM(COALESCE(r.dry, 0)) AS numeric), 2) as dry,
+                    ROUND(CAST(SUM(COALESCE(r.wet, 0)) AS numeric), 2) as wet
+                FROM date_range dr
+                LEFT JOIN newbook_net_revenue_data r ON r.date = dr.d
+                GROUP BY date_trunc('month', d)
+            )
+            SELECT
+                to_char(month_start, 'YYYY-MM-DD') as period_date,
+                accommodation,
+                dry,
+                wet
+            FROM monthly_data
+            ORDER BY month_start
+        """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    data_points = []
+    for row in rows:
+        accom = float(row.accommodation) if row.accommodation else 0.0
+        dry = float(row.dry) if row.dry else 0.0
+        wet = float(row.wet) if row.wet else 0.0
+
+        data_points.append(RevenueDataPoint(
+            date=row.period_date,
+            accommodation=accom,
+            dry=dry,
+            wet=wet,
+            total=round(accom + dry + wet, 2),
+        ))
+
+    return data_points
+
+
+# ============================================
+# RESTAURANT REPORTS
+# ============================================
+
+class ResosBookingsDataPoint(BaseModel):
+    date: str
+    total_bookings: int = 0
+    breakfast_bookings: int = 0
+    lunch_bookings: int = 0
+    afternoon_bookings: int = 0
+    dinner_bookings: int = 0
+    other_bookings: int = 0
+
+
+class ResosCoversDataPoint(BaseModel):
+    date: str
+    total_covers: int = 0
+    breakfast_covers: int = 0
+    lunch_covers: int = 0
+    afternoon_covers: int = 0
+    dinner_covers: int = 0
+    other_covers: int = 0
+    hotel_guest_covers: int = 0
+    non_hotel_guest_covers: int = 0
+
+
+@router.get("/restaurant-bookings", response_model=List[ResosBookingsDataPoint])
+async def get_restaurant_bookings_report(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    consolidation: str = Query("day", description="Consolidation period: day, week, or month"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get restaurant bookings data for the specified date range with optional consolidation.
+    """
+    # Validate dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    if consolidation not in ["day", "week", "month"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day', 'week', or 'month'")
+
+    if consolidation == "day":
+        query = text("""
+            SELECT
+                to_char(d, 'YYYY-MM-DD') as period_date,
+                COALESCE(s.total_bookings, 0) as total_bookings,
+                COALESCE(s.breakfast_bookings, 0) as breakfast_bookings,
+                COALESCE(s.lunch_bookings, 0) as lunch_bookings,
+                COALESCE(s.afternoon_bookings, 0) as afternoon_bookings,
+                COALESCE(s.dinner_bookings, 0) as dinner_bookings,
+                COALESCE(s.other_bookings, 0) as other_bookings
+            FROM generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) d
+            LEFT JOIN resos_bookings_stats s ON s.date = d
+            ORDER BY d
+        """)
+    elif consolidation == "week":
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            weekly_data AS (
+                SELECT
+                    CAST(date_trunc('week', d) AS date) as week_start,
+                    CAST(SUM(COALESCE(s.total_bookings, 0)) AS integer) as total_bookings,
+                    CAST(SUM(COALESCE(s.breakfast_bookings, 0)) AS integer) as breakfast_bookings,
+                    CAST(SUM(COALESCE(s.lunch_bookings, 0)) AS integer) as lunch_bookings,
+                    CAST(SUM(COALESCE(s.afternoon_bookings, 0)) AS integer) as afternoon_bookings,
+                    CAST(SUM(COALESCE(s.dinner_bookings, 0)) AS integer) as dinner_bookings,
+                    CAST(SUM(COALESCE(s.other_bookings, 0)) AS integer) as other_bookings
+                FROM date_range dr
+                LEFT JOIN resos_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('week', d)
+            )
+            SELECT
+                to_char(week_start, 'YYYY-MM-DD') as period_date,
+                total_bookings, breakfast_bookings, lunch_bookings,
+                afternoon_bookings, dinner_bookings, other_bookings
+            FROM weekly_data
+            ORDER BY week_start
+        """)
+    else:  # month
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            monthly_data AS (
+                SELECT
+                    CAST(date_trunc('month', d) AS date) as month_start,
+                    CAST(SUM(COALESCE(s.total_bookings, 0)) AS integer) as total_bookings,
+                    CAST(SUM(COALESCE(s.breakfast_bookings, 0)) AS integer) as breakfast_bookings,
+                    CAST(SUM(COALESCE(s.lunch_bookings, 0)) AS integer) as lunch_bookings,
+                    CAST(SUM(COALESCE(s.afternoon_bookings, 0)) AS integer) as afternoon_bookings,
+                    CAST(SUM(COALESCE(s.dinner_bookings, 0)) AS integer) as dinner_bookings,
+                    CAST(SUM(COALESCE(s.other_bookings, 0)) AS integer) as other_bookings
+                FROM date_range dr
+                LEFT JOIN resos_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('month', d)
+            )
+            SELECT
+                to_char(month_start, 'YYYY-MM-DD') as period_date,
+                total_bookings, breakfast_bookings, lunch_bookings,
+                afternoon_bookings, dinner_bookings, other_bookings
+            FROM monthly_data
+            ORDER BY month_start
+        """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    data_points = []
+    for row in rows:
+        data_points.append(ResosBookingsDataPoint(
+            date=row.period_date,
+            total_bookings=row.total_bookings or 0,
+            breakfast_bookings=row.breakfast_bookings or 0,
+            lunch_bookings=row.lunch_bookings or 0,
+            afternoon_bookings=row.afternoon_bookings or 0,
+            dinner_bookings=row.dinner_bookings or 0,
+            other_bookings=row.other_bookings or 0,
+        ))
+
+    return data_points
+
+
+@router.get("/restaurant-covers", response_model=List[ResosCoversDataPoint])
+async def get_restaurant_covers_report(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    consolidation: str = Query("day", description="Consolidation period: day, week, or month"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get restaurant covers (guests) data for the specified date range with optional consolidation.
+    """
+    # Validate dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    if consolidation not in ["day", "week", "month"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day', 'week', or 'month'")
+
+    if consolidation == "day":
+        query = text("""
+            SELECT
+                to_char(d, 'YYYY-MM-DD') as period_date,
+                COALESCE(s.total_covers, 0) as total_covers,
+                COALESCE(s.breakfast_covers, 0) as breakfast_covers,
+                COALESCE(s.lunch_covers, 0) as lunch_covers,
+                COALESCE(s.afternoon_covers, 0) as afternoon_covers,
+                COALESCE(s.dinner_covers, 0) as dinner_covers,
+                COALESCE(s.other_covers, 0) as other_covers,
+                COALESCE(s.hotel_guest_covers, 0) as hotel_guest_covers,
+                COALESCE(s.non_hotel_guest_covers, 0) as non_hotel_guest_covers
+            FROM generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) d
+            LEFT JOIN resos_bookings_stats s ON s.date = d
+            ORDER BY d
+        """)
+    elif consolidation == "week":
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            weekly_data AS (
+                SELECT
+                    CAST(date_trunc('week', d) AS date) as week_start,
+                    CAST(SUM(COALESCE(s.total_covers, 0)) AS integer) as total_covers,
+                    CAST(SUM(COALESCE(s.breakfast_covers, 0)) AS integer) as breakfast_covers,
+                    CAST(SUM(COALESCE(s.lunch_covers, 0)) AS integer) as lunch_covers,
+                    CAST(SUM(COALESCE(s.afternoon_covers, 0)) AS integer) as afternoon_covers,
+                    CAST(SUM(COALESCE(s.dinner_covers, 0)) AS integer) as dinner_covers,
+                    CAST(SUM(COALESCE(s.other_covers, 0)) AS integer) as other_covers,
+                    CAST(SUM(COALESCE(s.hotel_guest_covers, 0)) AS integer) as hotel_guest_covers,
+                    CAST(SUM(COALESCE(s.non_hotel_guest_covers, 0)) AS integer) as non_hotel_guest_covers
+                FROM date_range dr
+                LEFT JOIN resos_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('week', d)
+            )
+            SELECT
+                to_char(week_start, 'YYYY-MM-DD') as period_date,
+                total_covers, breakfast_covers, lunch_covers,
+                afternoon_covers, dinner_covers, other_covers,
+                hotel_guest_covers, non_hotel_guest_covers
+            FROM weekly_data
+            ORDER BY week_start
+        """)
+    else:  # month
+        query = text("""
+            WITH date_range AS (
+                SELECT CAST(generate_series(CAST(:start_date AS date), CAST(:end_date AS date), CAST('1 day' AS interval)) AS date) as d
+            ),
+            monthly_data AS (
+                SELECT
+                    CAST(date_trunc('month', d) AS date) as month_start,
+                    CAST(SUM(COALESCE(s.total_covers, 0)) AS integer) as total_covers,
+                    CAST(SUM(COALESCE(s.breakfast_covers, 0)) AS integer) as breakfast_covers,
+                    CAST(SUM(COALESCE(s.lunch_covers, 0)) AS integer) as lunch_covers,
+                    CAST(SUM(COALESCE(s.afternoon_covers, 0)) AS integer) as afternoon_covers,
+                    CAST(SUM(COALESCE(s.dinner_covers, 0)) AS integer) as dinner_covers,
+                    CAST(SUM(COALESCE(s.other_covers, 0)) AS integer) as other_covers,
+                    CAST(SUM(COALESCE(s.hotel_guest_covers, 0)) AS integer) as hotel_guest_covers,
+                    CAST(SUM(COALESCE(s.non_hotel_guest_covers, 0)) AS integer) as non_hotel_guest_covers
+                FROM date_range dr
+                LEFT JOIN resos_bookings_stats s ON s.date = dr.d
+                GROUP BY date_trunc('month', d)
+            )
+            SELECT
+                to_char(month_start, 'YYYY-MM-DD') as period_date,
+                total_covers, breakfast_covers, lunch_covers,
+                afternoon_covers, dinner_covers, other_covers,
+                hotel_guest_covers, non_hotel_guest_covers
+            FROM monthly_data
+            ORDER BY month_start
+        """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    data_points = []
+    for row in rows:
+        data_points.append(ResosCoversDataPoint(
+            date=row.period_date,
+            total_covers=row.total_covers or 0,
+            breakfast_covers=row.breakfast_covers or 0,
+            lunch_covers=row.lunch_covers or 0,
+            afternoon_covers=row.afternoon_covers or 0,
+            dinner_covers=row.dinner_covers or 0,
+            other_covers=row.other_covers or 0,
+            hotel_guest_covers=row.hotel_guest_covers or 0,
+            non_hotel_guest_covers=row.non_hotel_guest_covers or 0,
+        ))
+
+    return data_points
+
+
+# ============================================
+# 3D PICKUP VISUALIZATION ENDPOINT
+# ============================================
+
+class Pickup3DResponse(BaseModel):
+    """Response model for 3D pickup visualization data"""
+    start_date: str
+    end_date: str
+    metric: str
+    consolidation: str
+    arrival_dates: List[str]  # X-axis: dates in range
+    lead_times: List[int]     # Y-axis: lead times (days out)
+    surface_data: List[List[Optional[float]]]  # Z-axis: [lead_time][arrival_date] values
+    final_values: List[Optional[float]]  # Final values (d0) for each arrival date
+
+
+@router.get("/pickup-3d", response_model=Pickup3DResponse)
+async def get_pickup_3d_data(
+    start_date: str = Query(..., description="Start date (YYYY-MM-DD)"),
+    end_date: str = Query(..., description="End date (YYYY-MM-DD)"),
+    metric: str = Query("rooms", description="Metric: rooms or occupancy"),
+    consolidation: str = Query("day", description="Consolidation: day or week"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get 3D pickup visualization data for a date range.
+
+    Returns booking pace data structured for a 3D surface plot:
+    - X-axis: Arrival dates in the range
+    - Y-axis: Lead time (days before arrival when booking count was recorded)
+    - Z-axis: Room count or occupancy percentage
+
+    This visualizes how bookings accumulated over time for each arrival date.
+    """
+    if metric not in ["rooms", "occupancy"]:
+        raise HTTPException(status_code=400, detail="Metric must be 'rooms' or 'occupancy'")
+
+    if consolidation not in ["day", "week"]:
+        raise HTTPException(status_code=400, detail="Consolidation must be 'day' or 'week'")
+
+    # Parse dates
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
+
+    if start > end:
+        raise HTTPException(status_code=400, detail="Start date must be before end date")
+
+    # Define the lead times we want to show
+    # We'll show a reasonable subset to keep the visualization manageable
+    # Daily for d0-d30, weekly for d30-d90, monthly beyond that
+    lead_time_columns = [
+        # Daily (0-30 days)
+        'd0', 'd1', 'd2', 'd3', 'd4', 'd5', 'd6', 'd7',
+        'd8', 'd9', 'd10', 'd11', 'd12', 'd13', 'd14',
+        'd15', 'd16', 'd17', 'd18', 'd19', 'd20', 'd21',
+        'd22', 'd23', 'd24', 'd25', 'd26', 'd27', 'd28', 'd29', 'd30',
+        # Weekly (37-93 days)
+        'd37', 'd44', 'd51', 'd58', 'd65', 'd72', 'd79', 'd86', 'd93',
+        # Further out (100+ days)
+        'd100', 'd107', 'd114', 'd121', 'd128', 'd135', 'd142', 'd149',
+        'd156', 'd163', 'd170', 'd177',
+        # Monthly intervals
+        'd210', 'd240', 'd270', 'd300', 'd330', 'd365',
+    ]
+
+    # Build select clause for available columns
+    select_cols = ", ".join([f"COALESCE({col}, 0) as {col}" for col in lead_time_columns])
+
+    query = text(f"""
+        SELECT
+            arrival_date,
+            {select_cols}
+        FROM newbook_booking_pace
+        WHERE arrival_date >= :start_date AND arrival_date <= :end_date
+        ORDER BY arrival_date
+    """)
+
+    result = await db.execute(query, {"start_date": start, "end_date": end})
+    rows = result.fetchall()
+
+    # Generate all dates in range
+    all_dates = []
+    current = start
+    while current <= end:
+        all_dates.append(current.strftime("%Y-%m-%d"))
+        current += timedelta(days=1)
+
+    if not rows:
+        # Return empty structure if no data
+        lead_times = [int(col[1:]) for col in lead_time_columns]
+        return Pickup3DResponse(
+            start_date=start_date,
+            end_date=end_date,
+            metric=metric,
+            consolidation=consolidation,
+            arrival_dates=all_dates,
+            lead_times=lead_times,
+            surface_data=[[None] * len(all_dates) for _ in lead_times],
+            final_values=[None] * len(all_dates)
+        )
+
+    # Get total rooms for occupancy calculation
+    total_rooms = 27  # Hotel Number Four has 27 rooms
+
+    # Build the surface data
+    row_data_dict = {}
+    for row in rows:
+        arr_date = row.arrival_date.strftime("%Y-%m-%d")
+        row_data_dict[arr_date] = {col: getattr(row, col, 0) or 0 for col in lead_time_columns}
+
+    # Handle weekly consolidation
+    if consolidation == "week":
+        # Group dates by week (Monday start)
+        weekly_dates = []
+        weekly_data = {}
+        current = start
+        week_start = None
+
+        while current <= end:
+            # Get Monday of this week
+            days_since_monday = current.weekday()
+            monday = current - timedelta(days=days_since_monday)
+            week_label = monday.strftime("%Y-%m-%d")
+
+            if week_label not in weekly_data:
+                weekly_dates.append(week_label)
+                weekly_data[week_label] = {col: [] for col in lead_time_columns}
+
+            date_str = current.strftime("%Y-%m-%d")
+            if date_str in row_data_dict:
+                for col in lead_time_columns:
+                    weekly_data[week_label][col].append(row_data_dict[date_str].get(col, 0) or 0)
+
+            current += timedelta(days=1)
+
+        # Average the weekly data
+        all_dates = weekly_dates
+        row_data_dict = {}
+        for week_label in weekly_dates:
+            row_data_dict[week_label] = {}
+            for col in lead_time_columns:
+                values = weekly_data[week_label][col]
+                if values:
+                    row_data_dict[week_label][col] = sum(values) / len(values)
+                else:
+                    row_data_dict[week_label][col] = 0
+
+    # Build surface_data: [lead_time_index][arrival_date_index]
+    lead_times = [int(col[1:]) for col in lead_time_columns]
+    surface_data = []
+
+    for col in lead_time_columns:
+        lead_row = []
+        for arr_date in all_dates:
+            if arr_date in row_data_dict:
+                value = row_data_dict[arr_date].get(col, 0) or 0
+                if metric == "occupancy":
+                    # Convert to occupancy percentage
+                    value = round((value / total_rooms) * 100, 1) if total_rooms > 0 else 0
+                lead_row.append(value)
+            else:
+                lead_row.append(None)
+        surface_data.append(lead_row)
+
+    # Get final values (d0) for each arrival date
+    final_values = []
+    for arr_date in all_dates:
+        if arr_date in row_data_dict:
+            value = row_data_dict[arr_date].get('d0', 0) or 0
+            if metric == "occupancy":
+                value = round((value / total_rooms) * 100, 1) if total_rooms > 0 else 0
+            final_values.append(value)
+        else:
+            final_values.append(None)
+
+    return Pickup3DResponse(
+        start_date=start_date,
+        end_date=end_date,
+        metric=metric,
+        consolidation=consolidation,
+        arrival_dates=all_dates,
+        lead_times=lead_times,
+        surface_data=surface_data,
+        final_values=final_values
+    )

backend/api/resos.py

@@ -0,0 +1,718 @@
+"""
+Resos Mapping API Endpoints
+
+Handles Resos custom field and opening hours configuration.
+"""
+import logging
+from typing import Optional, List
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+
+from database import get_db
+from auth import get_current_user
+from services.resos_client import ResosClient, ResosAPIError
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+# ============ Pydantic Schemas ============
+
+class CustomFieldMappingItem(BaseModel):
+    field_id: str
+    field_name: str
+    field_type: str
+    maps_to: str  # 'hotel_guest', 'dbb', 'booking_number', 'allergies', 'ignore'
+    value_for_true: Optional[str] = None  # For radio/checkbox: which value means "yes"
+
+
+class OpeningHoursMappingItem(BaseModel):
+    opening_hour_id: str
+    opening_hour_name: str
+    period_type: str  # 'lunch', 'afternoon', 'dinner', 'ignore'
+    is_regular: bool = True
+
+
+class MappingUpdate(BaseModel):
+    custom_fields: Optional[List[CustomFieldMappingItem]] = None
+    opening_hours: Optional[List[OpeningHoursMappingItem]] = None
+
+
+# ============ Custom Fields Endpoints ============
+
+@router.get("/custom-fields")
+async def fetch_custom_fields(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Fetch custom field definitions from Resos API.
+    Returns fields that can be mapped to hotel_guest, dbb, etc.
+    """
+    try:
+        async with await ResosClient.from_db(db) as client:
+            fields = await client.get_custom_field_definitions()
+    except ResosAPIError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Failed to fetch Resos custom fields: {e}")
+        raise HTTPException(status_code=500, detail="Failed to fetch custom fields from Resos")
+
+    # Return all custom fields (no filtering by type - let user decide what to map)
+    formatted_fields = [
+        {
+            "id": f.get("_id") or f.get("id"),
+            "name": f.get("name", ""),
+            "type": f.get("type", ""),
+            "values": f.get("choices", [])  # For radio/dropdown fields
+        }
+        for f in fields
+    ]
+
+    return formatted_fields
+
+
+@router.get("/opening-hours")
+async def fetch_opening_hours(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Fetch opening hours/service periods from Resos API.
+    Filters to regular (non-special) periods only.
+    """
+    try:
+        async with await ResosClient.from_db(db) as client:
+            hours = await client.get_opening_hours()
+    except ResosAPIError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Failed to fetch Resos opening hours: {e}")
+        raise HTTPException(status_code=500, detail="Failed to fetch opening hours from Resos")
+
+    logger.info(f"Raw opening hours from Resos API: {len(hours)} periods")
+
+    # Filter out special/one-off periods - only return regular service periods
+    # special=True means one-off events, special=False means recurring
+    regular_hours = [h for h in hours if h.get('special') == False]
+
+    # Day of week mapping (Resos uses 1=Monday, 7=Sunday)
+    day_names = ['', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
+
+    # Transform and add helpful fields
+    formatted_hours = []
+    for hour in regular_hours:
+        hour_id = hour.get('_id') or hour.get('id', '')
+        hour_name = hour.get('name', '')
+        day_num = hour.get('day', 0)
+
+        # Convert open/close times from HHMM integers to HH:MM strings
+        start_time = None
+        end_time = None
+
+        if 'open' in hour:
+            open_val = hour['open']
+            hours_part = open_val // 100
+            mins_part = open_val % 100
+            start_time = f"{hours_part:02d}:{mins_part:02d}"
+
+        if 'close' in hour:
+            close_val = hour['close']
+            hours_part = close_val // 100
+            mins_part = close_val % 100
+            end_time = f"{hours_part:02d}:{mins_part:02d}"
+
+        formatted_hours.append({
+            "id": hour_id,
+            "name": hour_name,
+            "day": day_num,
+            "day_name": day_names[day_num] if 1 <= day_num <= 7 else "Unknown",
+            "start_time": start_time,
+            "end_time": end_time
+        })
+
+    # Sort by day of week first, then by open time
+    formatted_hours.sort(key=lambda h: (h.get('day', 0), h.get('start_time', '')))
+
+    logger.info(f"After filtering: {len(formatted_hours)} regular periods (filtered out {len(hours) - len(formatted_hours)} special periods)")
+
+    return formatted_hours
+
+
+# ============ Mapping Storage Endpoints ============
+
+@router.get("/custom-field-mapping")
+async def get_custom_field_mappings(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get saved custom field mappings."""
+    cf_result = await db.execute(text("""
+        SELECT field_id, field_name, field_type, maps_to, value_for_true
+        FROM resos_custom_field_mapping
+        ORDER BY field_name
+    """))
+    custom_field_rows = cf_result.fetchall()
+
+    return [
+        {
+            "custom_field_id": row.field_id,
+            "mapping_type": row.maps_to,
+            "value_for_true": row.value_for_true
+        }
+        for row in custom_field_rows
+    ]
+
+
+@router.get("/opening-hours-mapping")
+async def get_opening_hours_mappings(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get saved opening hours mappings."""
+    oh_result = await db.execute(text("""
+        SELECT opening_hour_id, opening_hour_name, period_type, display_name, is_regular
+        FROM resos_opening_hours_mapping
+        ORDER BY opening_hour_name
+    """))
+    opening_hour_rows = oh_result.fetchall()
+
+    return [
+        {
+            "opening_hour_id": row.opening_hour_id,
+            "opening_hour_name": row.opening_hour_name,
+            "period_type": row.period_type,
+            "display_name": row.display_name,
+            "is_regular": row.is_regular
+        }
+        for row in opening_hour_rows
+    ]
+
+
+@router.get("/manual-breakfast-periods")
+async def get_manual_breakfast_periods(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get manual breakfast period configuration."""
+    # Get enabled flag
+    result = await db.execute(text("""
+        SELECT config_value FROM system_config WHERE config_key = 'resos_enable_manual_breakfast'
+    """))
+    row = result.fetchone()
+    enabled = row.config_value.lower() == 'true' if row and row.config_value else False
+
+    # Get periods
+    result = await db.execute(text("""
+        SELECT day_of_week, start_time, end_time, is_active
+        FROM resos_manual_breakfast_periods
+        ORDER BY day_of_week
+    """))
+    rows = result.fetchall()
+
+    periods = [
+        {
+            "day_of_week": row.day_of_week,
+            "start_time": str(row.start_time) if row.start_time else None,
+            "end_time": str(row.end_time) if row.end_time else None,
+            "is_active": row.is_active
+        }
+        for row in rows
+    ]
+
+    return {
+        "enabled": enabled,
+        "periods": periods
+    }
+
+
+@router.post("/custom-field-mapping")
+async def save_custom_field_mappings(
+    data: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Save custom field mappings."""
+    mappings = data.get("mappings", [])
+    saved = 0
+
+    for mapping in mappings:
+        field_id = mapping.get("custom_field_id")
+        mapping_type = mapping.get("mapping_type")
+        value_for_true = mapping.get("value_for_true")
+
+        if not field_id or not mapping_type:
+            continue
+
+        await db.execute(text("""
+            INSERT INTO resos_custom_field_mapping
+                (field_id, field_name, field_type, maps_to, value_for_true, updated_at)
+            VALUES
+                (:field_id, '', '', :maps_to, :value_for_true, NOW())
+            ON CONFLICT (field_id) DO UPDATE SET
+                maps_to = EXCLUDED.maps_to,
+                value_for_true = EXCLUDED.value_for_true,
+                updated_at = NOW()
+        """), {
+            "field_id": field_id,
+            "maps_to": mapping_type,
+            "value_for_true": value_for_true
+        })
+        saved += 1
+
+    await db.commit()
+    logger.info(f"Saved {saved} custom field mappings")
+
+    return {
+        "message": "Custom field mappings saved successfully",
+        "saved": saved
+    }
+
+
+@router.post("/opening-hours-mapping")
+async def save_opening_hours_mappings(
+    data: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Save opening hours mappings."""
+    mappings = data.get("mappings", [])
+    saved = 0
+
+    for mapping in mappings:
+        opening_hour_id = mapping.get("opening_hour_id")
+        period_type = mapping.get("period_type")
+        display_name = mapping.get("display_name")
+
+        if not opening_hour_id or not period_type:
+            continue
+
+        await db.execute(text("""
+            INSERT INTO resos_opening_hours_mapping
+                (opening_hour_id, opening_hour_name, period_type, display_name, is_regular, updated_at)
+            VALUES
+                (:opening_hour_id, '', :period_type, :display_name, TRUE, NOW())
+            ON CONFLICT (opening_hour_id) DO UPDATE SET
+                period_type = EXCLUDED.period_type,
+                display_name = EXCLUDED.display_name,
+                updated_at = NOW()
+        """), {
+            "opening_hour_id": opening_hour_id,
+            "period_type": period_type,
+            "display_name": display_name
+        })
+        saved += 1
+
+    await db.commit()
+    logger.info(f"Saved {saved} opening hours mappings")
+
+    return {
+        "message": "Opening hours mappings saved successfully",
+        "saved": saved
+    }
+
+
+@router.post("/manual-breakfast-periods")
+async def save_manual_breakfast_periods(
+    data: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Save manual breakfast period configuration."""
+    from datetime import time as dt_time
+
+    enabled = data.get("enabled", False)
+    periods = data.get("periods", [])
+
+    # Save enabled flag to system_config
+    await db.execute(text("""
+        INSERT INTO system_config (config_key, config_value, description, updated_at)
+        VALUES ('resos_enable_manual_breakfast', :value, 'Enable manual breakfast configuration', NOW())
+        ON CONFLICT (config_key) DO UPDATE SET
+            config_value = EXCLUDED.config_value,
+            updated_at = NOW()
+    """), {"value": str(enabled).lower()})
+
+    # Save periods
+    for period in periods:
+        # Convert time strings to Python time objects for asyncpg
+        start_time_str = period.get("start_time")
+        end_time_str = period.get("end_time")
+
+        start_time_obj = None
+        end_time_obj = None
+
+        if start_time_str:
+            hour, minute = map(int, start_time_str.split(':'))
+            start_time_obj = dt_time(hour, minute)
+
+        if end_time_str:
+            hour, minute = map(int, end_time_str.split(':'))
+            end_time_obj = dt_time(hour, minute)
+
+        await db.execute(text("""
+            INSERT INTO resos_manual_breakfast_periods
+                (day_of_week, start_time, end_time, is_active, updated_at)
+            VALUES
+                (:day_of_week, :start_time, :end_time, :is_active, NOW())
+            ON CONFLICT (day_of_week) DO UPDATE SET
+                start_time = EXCLUDED.start_time,
+                end_time = EXCLUDED.end_time,
+                is_active = EXCLUDED.is_active,
+                updated_at = NOW()
+        """), {
+            "day_of_week": period["day_of_week"],
+            "start_time": start_time_obj,
+            "end_time": end_time_obj,
+            "is_active": period.get("is_active", True)
+        })
+
+    await db.commit()
+    return {"message": "Manual breakfast periods saved successfully"}
+
+
+@router.get("/mapping")
+async def get_mappings(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get saved custom field and opening hours mappings.
+    """
+    # Get custom field mappings
+    cf_result = await db.execute(text("""
+        SELECT field_id, field_name, field_type, maps_to, value_for_true
+        FROM resos_custom_field_mapping
+        ORDER BY field_name
+    """))
+    custom_field_rows = cf_result.fetchall()
+
+    # Get opening hours mappings
+    oh_result = await db.execute(text("""
+        SELECT opening_hour_id, opening_hour_name, period_type, is_regular
+        FROM resos_opening_hours_mapping
+        ORDER BY opening_hour_name
+    """))
+    opening_hour_rows = oh_result.fetchall()
+
+    return {
+        "custom_fields": [
+            {
+                "field_id": row.field_id,
+                "field_name": row.field_name,
+                "field_type": row.field_type,
+                "maps_to": row.maps_to,
+                "value_for_true": row.value_for_true
+            }
+            for row in custom_field_rows
+        ],
+        "opening_hours": [
+            {
+                "opening_hour_id": row.opening_hour_id,
+                "opening_hour_name": row.opening_hour_name,
+                "period_type": row.period_type,
+                "is_regular": row.is_regular
+            }
+            for row in opening_hour_rows
+        ]
+    }
+
+
+@router.post("/mapping")
+async def save_mappings(
+    update: MappingUpdate,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Save custom field and opening hours mappings.
+    Uses upsert to update existing or insert new mappings.
+    """
+    saved_cf = 0
+    saved_oh = 0
+
+    # Save custom field mappings
+    if update.custom_fields:
+        for cf in update.custom_fields:
+            await db.execute(text("""
+                INSERT INTO resos_custom_field_mapping
+                    (field_id, field_name, field_type, maps_to, value_for_true, updated_at)
+                VALUES
+                    (:field_id, :field_name, :field_type, :maps_to, :value_for_true, NOW())
+                ON CONFLICT (field_id) DO UPDATE SET
+                    field_name = EXCLUDED.field_name,
+                    field_type = EXCLUDED.field_type,
+                    maps_to = EXCLUDED.maps_to,
+                    value_for_true = EXCLUDED.value_for_true,
+                    updated_at = NOW()
+            """), {
+                "field_id": cf.field_id,
+                "field_name": cf.field_name,
+                "field_type": cf.field_type,
+                "maps_to": cf.maps_to,
+                "value_for_true": cf.value_for_true
+            })
+            saved_cf += 1
+
+    # Save opening hours mappings
+    if update.opening_hours:
+        for oh in update.opening_hours:
+            await db.execute(text("""
+                INSERT INTO resos_opening_hours_mapping
+                    (opening_hour_id, opening_hour_name, period_type, is_regular, updated_at)
+                VALUES
+                    (:opening_hour_id, :opening_hour_name, :period_type, :is_regular, NOW())
+                ON CONFLICT (opening_hour_id) DO UPDATE SET
+                    opening_hour_name = EXCLUDED.opening_hour_name,
+                    period_type = EXCLUDED.period_type,
+                    is_regular = EXCLUDED.is_regular,
+                    updated_at = NOW()
+            """), {
+                "opening_hour_id": oh.opening_hour_id,
+                "opening_hour_name": oh.opening_hour_name,
+                "period_type": oh.period_type,
+                "is_regular": oh.is_regular
+            })
+            saved_oh += 1
+
+    await db.commit()
+
+    logger.info(f"Saved {saved_cf} custom field mappings, {saved_oh} opening hours mappings")
+
+    return {
+        "message": "Mappings saved successfully",
+        "custom_fields_saved": saved_cf,
+        "opening_hours_saved": saved_oh
+    }
+
+
+@router.delete("/mapping/custom-field/{field_id}")
+async def delete_custom_field_mapping(
+    field_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Delete a custom field mapping."""
+    await db.execute(text("""
+        DELETE FROM resos_custom_field_mapping WHERE field_id = :field_id
+    """), {"field_id": field_id})
+    await db.commit()
+    return {"message": "Mapping deleted"}
+
+
+@router.delete("/mapping/opening-hour/{opening_hour_id}")
+async def delete_opening_hour_mapping(
+    opening_hour_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Delete an opening hours mapping."""
+    await db.execute(text("""
+        DELETE FROM resos_opening_hours_mapping WHERE opening_hour_id = :opening_hour_id
+    """), {"opening_hour_id": opening_hour_id})
+    await db.commit()
+    return {"message": "Mapping deleted"}
+
+
+# ============ Average Spend Settings Endpoints ============
+
+@router.get("/average-spend")
+async def get_average_spend(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get average spend settings for breakfast, lunch and dinner."""
+    # Get values from system_config
+    result = await db.execute(text("""
+        SELECT config_key, config_value FROM system_config
+        WHERE config_key IN (
+            'resos_breakfast_food_spend',
+            'resos_breakfast_drinks_spend',
+            'resos_lunch_food_spend',
+            'resos_lunch_drinks_spend',
+            'resos_dinner_food_spend',
+            'resos_dinner_drinks_spend'
+        )
+    """))
+    rows = result.fetchall()
+
+    settings = {}
+    for row in rows:
+        key = row.config_key.replace('resos_', '')
+        try:
+            settings[key] = float(row.config_value) if row.config_value else 0
+        except (ValueError, TypeError):
+            settings[key] = 0
+
+    return {
+        "breakfast_food_spend": settings.get('breakfast_food_spend', 0),
+        "breakfast_drinks_spend": settings.get('breakfast_drinks_spend', 0),
+        "lunch_food_spend": settings.get('lunch_food_spend', 0),
+        "lunch_drinks_spend": settings.get('lunch_drinks_spend', 0),
+        "dinner_food_spend": settings.get('dinner_food_spend', 0),
+        "dinner_drinks_spend": settings.get('dinner_drinks_spend', 0)
+    }
+
+
+@router.post("/average-spend")
+async def save_average_spend(
+    data: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Save average spend settings for breakfast, lunch and dinner."""
+    settings = [
+        ('resos_breakfast_food_spend', data.get('breakfast_food_spend', 0), 'Average food spend per cover for breakfast'),
+        ('resos_breakfast_drinks_spend', data.get('breakfast_drinks_spend', 0), 'Average drinks spend per cover for breakfast'),
+        ('resos_lunch_food_spend', data.get('lunch_food_spend', 0), 'Average food spend per cover for lunch'),
+        ('resos_lunch_drinks_spend', data.get('lunch_drinks_spend', 0), 'Average drinks spend per cover for lunch'),
+        ('resos_dinner_food_spend', data.get('dinner_food_spend', 0), 'Average food spend per cover for dinner'),
+        ('resos_dinner_drinks_spend', data.get('dinner_drinks_spend', 0), 'Average drinks spend per cover for dinner'),
+    ]
+
+    for config_key, config_value, description in settings:
+        await db.execute(text("""
+            INSERT INTO system_config (config_key, config_value, description, updated_at)
+            VALUES (:config_key, :config_value, :description, NOW())
+            ON CONFLICT (config_key) DO UPDATE SET
+                config_value = EXCLUDED.config_value,
+                updated_at = NOW()
+        """), {
+            "config_key": config_key,
+            "config_value": str(config_value),
+            "description": description
+        })
+
+    await db.commit()
+    logger.info("Saved average spend settings")
+
+    return {"message": "Average spend settings saved successfully"}
+
+
+# ============ Pace Data Backfill Endpoint ============
+
+@router.post("/backfill-pace")
+async def backfill_pace_data(
+    start_date: str,
+    end_date: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Backfill resos_booking_pace table for historical dates.
+
+    This calculates pace snapshots using booking_placed timestamps
+    to reconstruct what was on the books at each lead time.
+
+    Args:
+        start_date: Start date (YYYY-MM-DD)
+        end_date: End date (YYYY-MM-DD)
+    """
+    from datetime import datetime, timedelta
+
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        return {"error": "Invalid date format. Use YYYY-MM-DD"}
+
+    if (end - start).days > 400:
+        return {"error": "Date range too large. Max 400 days at a time."}
+
+    # Valid statuses and pace intervals
+    VALID_STATUSES = ('approved', 'arrived', 'seated', 'left')
+    PACE_INTERVALS = [
+        365, 330, 300, 270, 240, 210,
+        177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
+        107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
+        30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
+        20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
+        10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+    ]
+
+    dates_processed = 0
+    current = start
+
+    while current <= end:
+        for pace_type in ['total', 'resident', 'non_resident']:
+            pace_values = {}
+
+            for days_out in PACE_INTERVALS:
+                snapshot_date = current - timedelta(days=days_out)
+
+                # Build query based on pace_type
+                if pace_type == 'total':
+                    result = await db.execute(
+                        text("""
+                            SELECT COALESCE(SUM(covers), 0) as total_covers
+                            FROM resos_bookings_data
+                            WHERE booking_date = :target_date
+                            AND status IN ('approved', 'arrived', 'seated', 'left')
+                            AND booking_placed <= :snapshot_date
+                        """),
+                        {"target_date": current, "snapshot_date": snapshot_date}
+                    )
+                elif pace_type == 'resident':
+                    result = await db.execute(
+                        text("""
+                            SELECT COALESCE(SUM(covers), 0) as total_covers
+                            FROM resos_bookings_data
+                            WHERE booking_date = :target_date
+                            AND status IN ('approved', 'arrived', 'seated', 'left')
+                            AND booking_placed <= :snapshot_date
+                            AND is_hotel_guest = true
+                        """),
+                        {"target_date": current, "snapshot_date": snapshot_date}
+                    )
+                else:  # non_resident
+                    result = await db.execute(
+                        text("""
+                            SELECT COALESCE(SUM(covers), 0) as total_covers
+                            FROM resos_bookings_data
+                            WHERE booking_date = :target_date
+                            AND status IN ('approved', 'arrived', 'seated', 'left')
+                            AND booking_placed <= :snapshot_date
+                            AND (is_hotel_guest = false OR is_hotel_guest IS NULL)
+                        """),
+                        {"target_date": current, "snapshot_date": snapshot_date}
+                    )
+
+                row = result.fetchone()
+                pace_values[f"d{days_out}"] = row.total_covers if row else 0
+
+            # Upsert pace record
+            columns = ", ".join(pace_values.keys())
+            placeholders = ", ".join([f":{k}" for k in pace_values.keys()])
+            updates = ", ".join([f"{k} = :{k}" for k in pace_values.keys()])
+
+            await db.execute(
+                text(f"""
+                    INSERT INTO resos_booking_pace (booking_date, pace_type, {columns}, updated_at)
+                    VALUES (:booking_date, :pace_type, {placeholders}, NOW())
+                    ON CONFLICT (booking_date, pace_type) DO UPDATE SET
+                        {updates},
+                        updated_at = NOW()
+                """),
+                {"booking_date": current, "pace_type": pace_type, **pace_values}
+            )
+
+        dates_processed += 1
+        current += timedelta(days=1)
+
+        # Commit every 30 days to avoid large transactions
+        if dates_processed % 30 == 0:
+            await db.commit()
+            logger.info(f"Backfill progress: {dates_processed} dates processed")
+
+    await db.commit()
+    logger.info(f"Pace backfill completed: {dates_processed} dates from {start_date} to {end_date}")
+
+    return {
+        "message": f"Pace data backfilled successfully",
+        "dates_processed": dates_processed,
+        "start_date": start_date,
+        "end_date": end_date
+    }

backend/api/resos_sync.py

@@ -0,0 +1,244 @@
+"""
+Resos Bookings Sync API Endpoints
+Pattern: Similar to sync_bookings.py but for Resos
+"""
+import logging
+from datetime import date, timedelta
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+
+from database import get_db
+from auth import get_current_user
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+class ResosSyncConfig(BaseModel):
+    """Auto sync configuration for Resos bookings"""
+    auto_sync_enabled: bool
+    sync_time: str = "05:05"  # HH:MM format
+
+
+@router.get("/resos-bookings/status")
+async def get_resos_sync_status(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get sync status for Resos bookings data including last sync info."""
+    # Get last successful sync
+    result = await db.execute(
+        text("""
+            SELECT id, sync_type, started_at, completed_at, status,
+                   records_fetched, records_created, records_updated,
+                   date_from, date_to, error_message, triggered_by
+            FROM sync_log
+            WHERE source = 'resos' AND sync_type = 'bookings_data'
+            AND status = 'success'
+            ORDER BY completed_at DESC
+            LIMIT 1
+        """)
+    )
+    last_success = result.fetchone()
+
+    # Get last sync (any status)
+    result = await db.execute(
+        text("""
+            SELECT id, sync_type, started_at, completed_at, status,
+                   records_fetched, records_created, records_updated,
+                   date_from, date_to, error_message, triggered_by
+            FROM sync_log
+            WHERE source = 'resos' AND sync_type = 'bookings_data'
+            ORDER BY started_at DESC
+            LIMIT 1
+        """)
+    )
+    last_sync = result.fetchone()
+
+    # Get auto sync config
+    result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'sync_resos_bookings_enabled'")
+    )
+    row = result.fetchone()
+    auto_enabled = row.config_value.lower() == 'true' if row and row.config_value else False
+
+    result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'sync_resos_bookings_time'")
+    )
+    row = result.fetchone()
+    sync_time = row.config_value if row and row.config_value else '05:05'
+
+    # Get total records
+    result = await db.execute(text("SELECT COUNT(*) as count FROM resos_bookings_data"))
+    total_records = result.fetchone().count
+
+    # Get date range
+    result = await db.execute(
+        text("SELECT MIN(booking_date) as min_date, MAX(booking_date) as max_date FROM resos_bookings_data")
+    )
+    date_range = result.fetchone()
+
+    return {
+        "last_successful_sync": {
+            "completed_at": last_success.completed_at.isoformat() if last_success and last_success.completed_at else None,
+            "records_fetched": last_success.records_fetched if last_success else None,
+            "records_created": last_success.records_created if last_success else None,
+            "records_updated": last_success.records_updated if last_success else None,
+            "date_from": last_success.date_from.isoformat() if last_success and last_success.date_from else None,
+            "date_to": last_success.date_to.isoformat() if last_success and last_success.date_to else None,
+            "triggered_by": last_success.triggered_by if last_success else None,
+        } if last_success else None,
+        "last_sync": {
+            "started_at": last_sync.started_at.isoformat() if last_sync and last_sync.started_at else None,
+            "completed_at": last_sync.completed_at.isoformat() if last_sync and last_sync.completed_at else None,
+            "status": last_sync.status if last_sync else None,
+            "records_fetched": last_sync.records_fetched if last_sync else None,
+            "date_from": last_sync.date_from.isoformat() if last_sync and last_sync.date_from else None,
+            "date_to": last_sync.date_to.isoformat() if last_sync and last_sync.date_to else None,
+            "error_message": last_sync.error_message if last_sync else None,
+            "triggered_by": last_sync.triggered_by if last_sync else None,
+        } if last_sync else None,
+        "auto_sync": {
+            "enabled": auto_enabled,
+            "time": sync_time
+        },
+        "total_records": total_records,
+        "data_range": {
+            "from": date_range.min_date.isoformat() if date_range and date_range.min_date else None,
+            "to": date_range.max_date.isoformat() if date_range and date_range.max_date else None
+        }
+    }
+
+
+@router.post("/resos-bookings/sync")
+async def trigger_resos_sync(
+    background_tasks: BackgroundTasks,
+    from_date: Optional[date] = Query(None, description="Start date (default: today - 365)"),
+    to_date: Optional[date] = Query(None, description="End date (default: today + 365)"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger a Resos bookings sync.
+    Default: -365 to +365 days (historical + forecast window)
+    """
+    if not from_date:
+        from_date = date.today() - timedelta(days=365)
+    if not to_date:
+        to_date = date.today() + timedelta(days=365)
+
+    # Queue background task
+    background_tasks.add_task(
+        run_resos_sync_task,
+        from_date=from_date,
+        to_date=to_date,
+        triggered_by=f"user:{current_user['username']}"
+    )
+
+    return {
+        "status": "started",
+        "from_date": from_date.isoformat(),
+        "to_date": to_date.isoformat(),
+        "message": f"Resos bookings sync started for {from_date} to {to_date}"
+    }
+
+
+@router.get("/resos-bookings/config")
+async def get_resos_sync_config(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Get auto sync configuration for Resos bookings."""
+    # Get enabled setting
+    result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'sync_resos_bookings_enabled'")
+    )
+    row = result.fetchone()
+    enabled = row.config_value.lower() == 'true' if row and row.config_value else False
+
+    # Get sync time setting
+    result = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'sync_resos_bookings_time'")
+    )
+    row = result.fetchone()
+    sync_time = row.config_value if row and row.config_value else '05:05'
+
+    return {
+        "auto_sync_enabled": enabled,
+        "sync_time": sync_time
+    }
+
+
+@router.post("/resos-bookings/config")
+async def update_resos_sync_config(
+    config: ResosSyncConfig,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Update auto sync configuration for Resos bookings."""
+    # Upsert enabled setting
+    await db.execute(
+        text("""
+            INSERT INTO system_config (config_key, config_value, description, updated_at, updated_by)
+            VALUES ('sync_resos_bookings_enabled', :value, 'Enable automatic Resos bookings sync', NOW(), :user)
+            ON CONFLICT (config_key) DO UPDATE SET
+                config_value = :value,
+                updated_at = NOW(),
+                updated_by = :user
+        """),
+        {"value": str(config.auto_sync_enabled).lower(), "user": current_user['username']}
+    )
+
+    # Upsert sync time setting
+    await db.execute(
+        text("""
+            INSERT INTO system_config (config_key, config_value, description, updated_at, updated_by)
+            VALUES ('sync_resos_bookings_time', :value, 'Resos bookings sync time (HH:MM)', NOW(), :user)
+            ON CONFLICT (config_key) DO UPDATE SET
+                config_value = :value,
+                updated_at = NOW(),
+                updated_by = :user
+        """),
+        {"value": config.sync_time, "user": current_user['username']}
+    )
+
+    await db.commit()
+
+    return {
+        "status": "success",
+        "auto_sync_enabled": config.auto_sync_enabled,
+        "sync_time": config.sync_time
+    }
+
+
+def run_resos_sync_task(
+    from_date: date,
+    to_date: date,
+    triggered_by: str = "scheduler"
+):
+    """Background task to run Resos sync."""
+    import sys
+    import asyncio
+    from jobs.resos_bookings_sync import sync_resos_bookings_data
+
+    print(f"[SYNC-RESOS] Starting sync ({from_date} to {to_date})", flush=True)
+    sys.stdout.flush()
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    try:
+        loop.run_until_complete(
+            sync_resos_bookings_data(from_date, to_date, triggered_by)
+        )
+        print(f"[SYNC-RESOS] Sync completed", flush=True)
+    except Exception as e:
+        print(f"[SYNC-RESOS] FAILED: {e}", flush=True)
+        import traceback
+        traceback.print_exc()
+        raise
+    finally:
+        loop.close()

backend/api/special_dates.py

@@ -0,0 +1,571 @@
+"""
+Special Dates API - Configure custom holidays/events for Prophet forecasting
+"""
+from datetime import date, datetime, timedelta
+from typing import Optional, List
+from enum import Enum
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+
+from database import get_db
+from auth import get_current_user
+
+router = APIRouter()
+
+
+# ============================================
+# MODELS
+# ============================================
+
+class DatePatternType(str, Enum):
+    FIXED = "fixed"  # Fixed date each year (e.g., Feb 14)
+    NTH_WEEKDAY = "nth_weekday"  # Nth weekday of month (e.g., 2nd Monday of Feb)
+    RELATIVE_TO_DATE = "relative_to_date"  # Weekday before/after a fixed date
+
+
+class SpecialDateBase(BaseModel):
+    name: str
+    pattern_type: DatePatternType
+    # For FIXED pattern
+    fixed_month: Optional[int] = None  # 1-12
+    fixed_day: Optional[int] = None  # 1-31
+    # For NTH_WEEKDAY pattern
+    nth_week: Optional[int] = None  # 1-5 or -1 for last
+    weekday: Optional[int] = None  # 0=Mon, 6=Sun
+    month: Optional[int] = None  # 1-12
+    # For RELATIVE_TO_DATE pattern
+    relative_to_month: Optional[int] = None
+    relative_to_day: Optional[int] = None
+    relative_weekday: Optional[int] = None  # Which weekday to find
+    relative_direction: Optional[str] = None  # 'before' or 'after'
+    # Common fields
+    duration_days: int = 1
+    is_recurring: bool = True
+    one_off_year: Optional[int] = None  # Only if is_recurring = False
+    is_active: bool = True
+
+
+class SpecialDateCreate(SpecialDateBase):
+    pass
+
+
+class SpecialDateUpdate(SpecialDateBase):
+    pass
+
+
+class SpecialDateResponse(SpecialDateBase):
+    id: int
+    created_at: str
+
+
+class ResolvedDate(BaseModel):
+    name: str
+    date: str
+    day_of_week: str
+
+
+# ============================================
+# TABLE CREATION
+# ============================================
+
+async def ensure_table_exists(db: AsyncSession):
+    """Create the special_dates table if it doesn't exist"""
+    await db.execute(text("""
+        CREATE TABLE IF NOT EXISTS special_dates (
+            id SERIAL PRIMARY KEY,
+            name VARCHAR(100) NOT NULL,
+            pattern_type VARCHAR(20) NOT NULL,
+            fixed_month INTEGER,
+            fixed_day INTEGER,
+            nth_week INTEGER,
+            weekday INTEGER,
+            month INTEGER,
+            relative_to_month INTEGER,
+            relative_to_day INTEGER,
+            relative_weekday INTEGER,
+            relative_direction VARCHAR(10),
+            duration_days INTEGER DEFAULT 1,
+            is_recurring BOOLEAN DEFAULT TRUE,
+            one_off_year INTEGER,
+            is_active BOOLEAN DEFAULT TRUE,
+            created_at TIMESTAMP DEFAULT NOW(),
+            updated_at TIMESTAMP DEFAULT NOW()
+        )
+    """))
+    await db.commit()
+
+
+# ============================================
+# DATE RESOLUTION HELPERS
+# ============================================
+
+def get_nth_weekday_of_month(year: int, month: int, weekday: int, nth: int) -> Optional[date]:
+    """
+    Get the nth occurrence of a weekday in a month.
+    nth: 1-5 for first through fifth, -1 for last
+    weekday: 0=Monday, 6=Sunday
+    """
+    if nth == -1:
+        # Last occurrence - start from end of month
+        if month == 12:
+            next_month = date(year + 1, 1, 1)
+        else:
+            next_month = date(year, month + 1, 1)
+        last_day = next_month - timedelta(days=1)
+
+        # Find the last occurrence of the weekday
+        days_back = (last_day.weekday() - weekday) % 7
+        return last_day - timedelta(days=days_back)
+    else:
+        # Nth occurrence from start
+        first_of_month = date(year, month, 1)
+        # Find the first occurrence of the weekday
+        days_ahead = (weekday - first_of_month.weekday()) % 7
+        first_occurrence = first_of_month + timedelta(days=days_ahead)
+        # Add weeks to get to nth occurrence
+        result = first_occurrence + timedelta(weeks=nth - 1)
+        # Verify it's still in the same month
+        if result.month != month:
+            return None
+        return result
+
+
+def get_weekday_relative_to_date(year: int, month: int, day: int,
+                                  target_weekday: int, direction: str) -> Optional[date]:
+    """
+    Get a specific weekday before or after a fixed date.
+    target_weekday: 0=Monday, 6=Sunday
+    direction: 'before' or 'after'
+    """
+    try:
+        base_date = date(year, month, day)
+    except ValueError:
+        return None
+
+    if direction == 'before':
+        # Find the weekday before (or on) the base date
+        days_back = (base_date.weekday() - target_weekday) % 7
+        if days_back == 0:
+            days_back = 7  # If same weekday, go back a week
+        return base_date - timedelta(days=days_back)
+    else:  # after
+        # Find the weekday after the base date
+        days_ahead = (target_weekday - base_date.weekday()) % 7
+        if days_ahead == 0:
+            days_ahead = 7  # If same weekday, go forward a week
+        return base_date + timedelta(days=days_ahead)
+
+
+def resolve_special_date(sd: dict, year: int) -> List[date]:
+    """Resolve a special date pattern to actual dates for a given year"""
+    if not sd['is_recurring'] and sd.get('one_off_year') and sd['one_off_year'] != year:
+        return []
+
+    base_date = None
+    pattern_type = sd['pattern_type']
+
+    if pattern_type == 'fixed':
+        try:
+            base_date = date(year, sd['fixed_month'], sd['fixed_day'])
+        except (ValueError, TypeError):
+            return []
+
+    elif pattern_type == 'nth_weekday':
+        base_date = get_nth_weekday_of_month(
+            year, sd['month'], sd['weekday'], sd['nth_week']
+        )
+
+    elif pattern_type == 'relative_to_date':
+        base_date = get_weekday_relative_to_date(
+            year, sd['relative_to_month'], sd['relative_to_day'],
+            sd['relative_weekday'], sd['relative_direction']
+        )
+
+    if base_date is None:
+        return []
+
+    # Generate dates for duration
+    duration = sd.get('duration_days', 1) or 1
+    return [base_date + timedelta(days=i) for i in range(duration)]
+
+
+# ============================================
+# CRUD ENDPOINTS
+# ============================================
+
+@router.get("/special-dates", response_model=List[SpecialDateResponse])
+async def list_special_dates(
+    active_only: bool = Query(False, description="Only return active dates"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """List all special date configurations"""
+    await ensure_table_exists(db)
+
+    query = "SELECT * FROM special_dates"
+    if active_only:
+        query += " WHERE is_active = TRUE"
+    query += " ORDER BY name"
+
+    result = await db.execute(text(query))
+    rows = result.fetchall()
+
+    return [
+        SpecialDateResponse(
+            id=row.id,
+            name=row.name,
+            pattern_type=row.pattern_type,
+            fixed_month=row.fixed_month,
+            fixed_day=row.fixed_day,
+            nth_week=row.nth_week,
+            weekday=row.weekday,
+            month=row.month,
+            relative_to_month=row.relative_to_month,
+            relative_to_day=row.relative_to_day,
+            relative_weekday=row.relative_weekday,
+            relative_direction=row.relative_direction,
+            duration_days=row.duration_days or 1,
+            is_recurring=row.is_recurring,
+            one_off_year=row.one_off_year,
+            is_active=row.is_active,
+            created_at=str(row.created_at)
+        )
+        for row in rows
+    ]
+
+
+@router.post("/special-dates", response_model=SpecialDateResponse)
+async def create_special_date(
+    data: SpecialDateCreate,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Create a new special date configuration"""
+    await ensure_table_exists(db)
+
+    result = await db.execute(text("""
+        INSERT INTO special_dates (
+            name, pattern_type, fixed_month, fixed_day, nth_week, weekday, month,
+            relative_to_month, relative_to_day, relative_weekday, relative_direction,
+            duration_days, is_recurring, one_off_year, is_active
+        ) VALUES (
+            :name, :pattern_type, :fixed_month, :fixed_day, :nth_week, :weekday, :month,
+            :relative_to_month, :relative_to_day, :relative_weekday, :relative_direction,
+            :duration_days, :is_recurring, :one_off_year, :is_active
+        ) RETURNING id, created_at
+    """), {
+        "name": data.name,
+        "pattern_type": data.pattern_type.value,
+        "fixed_month": data.fixed_month,
+        "fixed_day": data.fixed_day,
+        "nth_week": data.nth_week,
+        "weekday": data.weekday,
+        "month": data.month,
+        "relative_to_month": data.relative_to_month,
+        "relative_to_day": data.relative_to_day,
+        "relative_weekday": data.relative_weekday,
+        "relative_direction": data.relative_direction,
+        "duration_days": data.duration_days,
+        "is_recurring": data.is_recurring,
+        "one_off_year": data.one_off_year,
+        "is_active": data.is_active
+    })
+
+    row = result.fetchone()
+    await db.commit()
+
+    return SpecialDateResponse(
+        id=row.id,
+        created_at=str(row.created_at),
+        **data.dict()
+    )
+
+
+@router.put("/special-dates/{date_id}", response_model=SpecialDateResponse)
+async def update_special_date(
+    date_id: int,
+    data: SpecialDateUpdate,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Update a special date configuration"""
+    result = await db.execute(text("""
+        UPDATE special_dates SET
+            name = :name,
+            pattern_type = :pattern_type,
+            fixed_month = :fixed_month,
+            fixed_day = :fixed_day,
+            nth_week = :nth_week,
+            weekday = :weekday,
+            month = :month,
+            relative_to_month = :relative_to_month,
+            relative_to_day = :relative_to_day,
+            relative_weekday = :relative_weekday,
+            relative_direction = :relative_direction,
+            duration_days = :duration_days,
+            is_recurring = :is_recurring,
+            one_off_year = :one_off_year,
+            is_active = :is_active,
+            updated_at = NOW()
+        WHERE id = :id
+        RETURNING id, created_at
+    """), {
+        "id": date_id,
+        "name": data.name,
+        "pattern_type": data.pattern_type.value,
+        "fixed_month": data.fixed_month,
+        "fixed_day": data.fixed_day,
+        "nth_week": data.nth_week,
+        "weekday": data.weekday,
+        "month": data.month,
+        "relative_to_month": data.relative_to_month,
+        "relative_to_day": data.relative_to_day,
+        "relative_weekday": data.relative_weekday,
+        "relative_direction": data.relative_direction,
+        "duration_days": data.duration_days,
+        "is_recurring": data.is_recurring,
+        "one_off_year": data.one_off_year,
+        "is_active": data.is_active
+    })
+
+    row = result.fetchone()
+    if not row:
+        raise HTTPException(status_code=404, detail="Special date not found")
+
+    await db.commit()
+
+    return SpecialDateResponse(
+        id=row.id,
+        created_at=str(row.created_at),
+        **data.dict()
+    )
+
+
+@router.delete("/special-dates/{date_id}")
+async def delete_special_date(
+    date_id: int,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Delete a special date configuration"""
+    result = await db.execute(
+        text("DELETE FROM special_dates WHERE id = :id RETURNING id"),
+        {"id": date_id}
+    )
+
+    if not result.fetchone():
+        raise HTTPException(status_code=404, detail="Special date not found")
+
+    await db.commit()
+    return {"message": "Special date deleted successfully"}
+
+
+@router.get("/special-dates/preview", response_model=List[ResolvedDate])
+async def preview_special_dates(
+    year: int = Query(..., description="Year to preview dates for"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Preview resolved dates for a given year"""
+    await ensure_table_exists(db)
+
+    result = await db.execute(text(
+        "SELECT * FROM special_dates WHERE is_active = TRUE ORDER BY name"
+    ))
+    rows = result.fetchall()
+
+    resolved = []
+    for row in rows:
+        sd = {
+            'pattern_type': row.pattern_type,
+            'fixed_month': row.fixed_month,
+            'fixed_day': row.fixed_day,
+            'nth_week': row.nth_week,
+            'weekday': row.weekday,
+            'month': row.month,
+            'relative_to_month': row.relative_to_month,
+            'relative_to_day': row.relative_to_day,
+            'relative_weekday': row.relative_weekday,
+            'relative_direction': row.relative_direction,
+            'duration_days': row.duration_days,
+            'is_recurring': row.is_recurring,
+            'one_off_year': row.one_off_year
+        }
+
+        dates = resolve_special_date(sd, year)
+        for d in dates:
+            resolved.append(ResolvedDate(
+                name=row.name,
+                date=str(d),
+                day_of_week=d.strftime("%a")
+            ))
+
+    # Sort by date
+    resolved.sort(key=lambda x: x.date)
+    return resolved
+
+
+@router.post("/special-dates/seed-defaults")
+async def seed_default_dates(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """Seed default special dates (Valentine's, Christmas Eve, NYE, Bank Holiday weekends)"""
+    await ensure_table_exists(db)
+
+    # Check if already seeded
+    result = await db.execute(text("SELECT COUNT(*) as count FROM special_dates"))
+    if result.fetchone().count > 0:
+        return {"message": "Special dates already exist. Delete existing to re-seed."}
+
+    defaults = [
+        # Valentine's Day
+        {
+            "name": "Valentine's Day",
+            "pattern_type": "fixed",
+            "fixed_month": 2,
+            "fixed_day": 14,
+            "duration_days": 1
+        },
+        # Christmas Eve
+        {
+            "name": "Christmas Eve",
+            "pattern_type": "fixed",
+            "fixed_month": 12,
+            "fixed_day": 24,
+            "duration_days": 1
+        },
+        # New Year's Eve
+        {
+            "name": "New Year's Eve",
+            "pattern_type": "fixed",
+            "fixed_month": 12,
+            "fixed_day": 31,
+            "duration_days": 1
+        },
+        # Friday before Christmas (if Christmas is Sat-Tue)
+        {
+            "name": "Friday Before Christmas",
+            "pattern_type": "relative_to_date",
+            "relative_to_month": 12,
+            "relative_to_day": 25,
+            "relative_weekday": 4,  # Friday
+            "relative_direction": "before",
+            "duration_days": 1
+        },
+        # Saturday before Christmas
+        {
+            "name": "Saturday Before Christmas",
+            "pattern_type": "relative_to_date",
+            "relative_to_month": 12,
+            "relative_to_day": 25,
+            "relative_weekday": 5,  # Saturday
+            "relative_direction": "before",
+            "duration_days": 1
+        },
+        # Early May Bank Holiday Weekend (Fri-Sat of first Monday in May)
+        {
+            "name": "Early May BH Friday",
+            "pattern_type": "nth_weekday",
+            "month": 5,
+            "nth_week": 1,
+            "weekday": 4,  # Friday (before the Monday)
+            "duration_days": 1
+        },
+        # Spring Bank Holiday Weekend (last Monday of May)
+        {
+            "name": "Spring BH Weekend",
+            "pattern_type": "nth_weekday",
+            "month": 5,
+            "nth_week": -1,  # Last week
+            "weekday": 5,  # Saturday
+            "duration_days": 2  # Sat + Sun (Mon is the bank holiday itself)
+        },
+        # August Bank Holiday Weekend (last Monday of August)
+        {
+            "name": "August BH Weekend",
+            "pattern_type": "nth_weekday",
+            "month": 8,
+            "nth_week": -1,
+            "weekday": 5,  # Saturday
+            "duration_days": 2
+        },
+    ]
+
+    for d in defaults:
+        await db.execute(text("""
+            INSERT INTO special_dates (
+                name, pattern_type, fixed_month, fixed_day, nth_week, weekday, month,
+                relative_to_month, relative_to_day, relative_weekday, relative_direction,
+                duration_days, is_recurring, is_active
+            ) VALUES (
+                :name, :pattern_type, :fixed_month, :fixed_day, :nth_week, :weekday, :month,
+                :relative_to_month, :relative_to_day, :relative_weekday, :relative_direction,
+                :duration_days, TRUE, TRUE
+            )
+        """), {
+            "name": d["name"],
+            "pattern_type": d["pattern_type"],
+            "fixed_month": d.get("fixed_month"),
+            "fixed_day": d.get("fixed_day"),
+            "nth_week": d.get("nth_week"),
+            "weekday": d.get("weekday"),
+            "month": d.get("month"),
+            "relative_to_month": d.get("relative_to_month"),
+            "relative_to_day": d.get("relative_to_day"),
+            "relative_weekday": d.get("relative_weekday"),
+            "relative_direction": d.get("relative_direction"),
+            "duration_days": d.get("duration_days", 1)
+        })
+
+    await db.commit()
+    return {"message": f"Seeded {len(defaults)} default special dates"}
+
+
+# ============================================
+# HELPER FOR PROPHET INTEGRATION
+# ============================================
+
+async def get_special_dates_for_prophet(db: AsyncSession, start_year: int, end_year: int) -> List[dict]:
+    """
+    Get all special dates resolved for a range of years, formatted for Prophet.
+    Returns list of dicts with 'ds' (date) and 'holiday' (name) columns.
+    """
+    await ensure_table_exists(db)
+
+    result = await db.execute(text(
+        "SELECT * FROM special_dates WHERE is_active = TRUE"
+    ))
+    rows = result.fetchall()
+
+    prophet_holidays = []
+    for year in range(start_year, end_year + 1):
+        for row in rows:
+            sd = {
+                'pattern_type': row.pattern_type,
+                'fixed_month': row.fixed_month,
+                'fixed_day': row.fixed_day,
+                'nth_week': row.nth_week,
+                'weekday': row.weekday,
+                'month': row.month,
+                'relative_to_month': row.relative_to_month,
+                'relative_to_day': row.relative_to_day,
+                'relative_weekday': row.relative_weekday,
+                'relative_direction': row.relative_direction,
+                'duration_days': row.duration_days,
+                'is_recurring': row.is_recurring,
+                'one_off_year': row.one_off_year
+            }
+
+            dates = resolve_special_date(sd, year)
+            for d in dates:
+                # Prophet expects datetime objects, not date objects
+                prophet_holidays.append({
+                    'ds': datetime.combine(d, datetime.min.time()),
+                    'holiday': row.name
+                })
+
+    return prophet_holidays

backend/api/sync.py

@@ -0,0 +1,888 @@
+"""
+Data Sync API endpoints
+"""
+import uuid
+import logging
+from datetime import date, timedelta
+from dateutil.relativedelta import relativedelta
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from pydantic import BaseModel
+
+from database import get_db, SyncSessionLocal
+from auth import get_current_user
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+class BackfillRequest(BaseModel):
+    """Request model for backfill job"""
+    source: str  # 'newbook', 'resos', or 'all'
+    from_date: date
+    to_date: date
+    chunk_months: int = 1  # Process in monthly chunks to avoid timeouts
+
+
+@router.get("/status")
+async def get_sync_status(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get last sync times and status for all data sources and sync types.
+
+    Returns dict keyed by "{source}_{sync_type}" (e.g., "newbook_bookings", "newbook_earned_revenue")
+    """
+    query = """
+        SELECT DISTINCT ON (source, sync_type)
+            source,
+            sync_type,
+            completed_at as last_sync,
+            status as last_status,
+            records_fetched as last_records,
+            records_created as last_created,
+            date_from,
+            date_to
+        FROM sync_log
+        ORDER BY source, sync_type, completed_at DESC
+    """
+
+    result = await db.execute(text(query))
+    rows = result.fetchall()
+
+    response = {}
+    for row in rows:
+        # Create key like "newbook_bookings" or "newbook_earned_revenue"
+        key = f"{row.source}_{row.sync_type}" if row.sync_type else row.source
+        response[key] = {
+            "source": row.source,
+            "sync_type": row.sync_type,
+            "last_sync": row.last_sync,
+            "status": row.last_status,
+            "records_fetched": row.last_records,
+            "records_created": row.last_created,
+            "date_from": row.date_from,
+            "date_to": row.date_to
+        }
+
+    return response
+
+
+@router.post("/newbook")
+async def trigger_newbook_sync(
+    background_tasks: BackgroundTasks,
+    full_sync: bool = Query(False, description="If True, fetches all bookings. If False, only fetches since last sync."),
+    from_date: Optional[date] = Query(None, description="Start date for stay period (filters by arrival/stay dates)"),
+    to_date: Optional[date] = Query(None, description="End date for stay period (filters by arrival/stay dates)"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger manual Newbook data sync.
+    Runs in background to avoid timeout.
+
+    - full_sync=False (default): Incremental sync - only bookings modified since last successful sync
+    - full_sync=True: Full sync - fetches entire booking database
+    - from_date/to_date: If provided, fetches bookings staying during this period (overrides full_sync)
+    """
+    from jobs.data_sync import sync_newbook_data
+
+    # Queue background task
+    background_tasks.add_task(
+        sync_newbook_data,
+        full_sync=full_sync,
+        from_date=from_date,
+        to_date=to_date,
+        triggered_by=f"user:{current_user['username']}"
+    )
+
+    msg = "Newbook sync started in background"
+    if from_date and to_date:
+        msg = f"Newbook sync for {from_date} to {to_date} started in background"
+    elif full_sync:
+        msg = "Newbook full sync started in background"
+    else:
+        msg = "Newbook incremental sync started in background"
+
+    return {
+        "status": "started",
+        "source": "newbook",
+        "full_sync": full_sync,
+        "from_date": from_date,
+        "to_date": to_date,
+        "message": msg
+    }
+
+
+@router.post("/resos")
+async def trigger_resos_sync(
+    background_tasks: BackgroundTasks,
+    from_date: Optional[date] = Query(None, description="Start date for sync"),
+    to_date: Optional[date] = Query(None, description="End date for sync"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger manual Resos data sync.
+    Runs in background to avoid timeout.
+    """
+    from jobs.data_sync import sync_resos_data
+
+    if from_date is None:
+        from_date = date.today() - timedelta(days=7)
+    if to_date is None:
+        to_date = date.today() + timedelta(days=365)
+
+    # Queue background task
+    background_tasks.add_task(
+        sync_resos_data,
+        from_date=from_date,
+        to_date=to_date,
+        triggered_by=f"user:{current_user['username']}"
+    )
+
+    return {
+        "status": "started",
+        "source": "resos",
+        "from_date": from_date,
+        "to_date": to_date,
+        "message": "Resos sync started in background"
+    }
+
+
+@router.post("/newbook/occupancy-report")
+async def trigger_occupancy_report_sync(
+    background_tasks: BackgroundTasks,
+    from_date: Optional[date] = Query(None, description="Start date for report"),
+    to_date: Optional[date] = Query(None, description="End date for report"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger Newbook occupancy report sync.
+
+    This fetches Newbook's official occupancy report which provides:
+    - Available rooms (accounting for maintenance/offline)
+    - Official occupied room counts
+    - Maintenance/offline room counts
+    - Official revenue figures (gross and net)
+
+    Use this to ensure accurate occupancy % calculations when rooms
+    have been taken offline for maintenance.
+    """
+    from jobs.data_sync import sync_newbook_occupancy_report
+
+    if from_date is None:
+        from_date = date.today() - timedelta(days=90)
+    if to_date is None:
+        to_date = date.today() + timedelta(days=30)
+
+    background_tasks.add_task(
+        sync_newbook_occupancy_report,
+        from_date=from_date,
+        to_date=to_date,
+        triggered_by=f"user:{current_user['username']}"
+    )
+
+    return {
+        "status": "started",
+        "source": "newbook_occupancy_report",
+        "from_date": from_date,
+        "to_date": to_date,
+        "message": f"Newbook occupancy report sync started for {from_date} to {to_date}"
+    }
+
+
+@router.post("/newbook/earned-revenue")
+async def trigger_earned_revenue_sync(
+    background_tasks: BackgroundTasks,
+    from_date: Optional[date] = Query(None, description="Start date for revenue"),
+    to_date: Optional[date] = Query(None, description="End date for revenue"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger Newbook earned revenue sync.
+
+    This fetches official financial figures by GL account from Newbook's
+    report_earned_revenue endpoint. Uses accommodation_gl_codes config
+    to identify which GL accounts are room revenue.
+
+    Defaults to last 7 days if no dates specified (catches adjustments).
+    For historical backfill, specify a wider date range.
+    """
+    from jobs.data_sync import sync_newbook_earned_revenue
+
+    if from_date is None:
+        from_date = date.today() - timedelta(days=7)
+    if to_date is None:
+        to_date = date.today()
+
+    background_tasks.add_task(
+        sync_newbook_earned_revenue,
+        from_date=from_date,
+        to_date=to_date,
+        triggered_by=f"user:{current_user['username']}"
+    )
+
+    return {
+        "status": "started",
+        "source": "newbook_earned_revenue",
+        "from_date": from_date,
+        "to_date": to_date,
+        "message": f"Newbook earned revenue sync started for {from_date} to {to_date}"
+    }
+
+
+@router.post("/full")
+async def trigger_full_sync(
+    background_tasks: BackgroundTasks,
+    full_sync: bool = Query(False, description="If True, fetches all bookings from both sources"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger full sync from all sources (Newbook bookings, Newbook occupancy report, Resos).
+    """
+    from jobs.data_sync import run_data_sync
+
+    background_tasks.add_task(
+        run_data_sync,
+        full_sync=full_sync,
+        triggered_by=f"user:{current_user['username']}"
+    )
+
+    return {
+        "status": "started",
+        "sources": ["newbook", "newbook_occupancy_report", "resos"],
+        "full_sync": full_sync,
+        "message": f"Full {'complete' if full_sync else 'incremental'} sync started in background"
+    }
+
+
+@router.post("/aggregate")
+async def trigger_aggregation(
+    background_tasks: BackgroundTasks,
+    source: Optional[str] = Query(None, description="Filter by source: newbook, resos. Leave empty for all."),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Trigger manual aggregation of pending dates.
+    Processes the aggregation queue and updates daily_occupancy/daily_covers tables.
+    """
+    from jobs.aggregation import run_aggregation
+
+    if source and source not in ['newbook', 'resos']:
+        raise HTTPException(status_code=400, detail="Source must be 'newbook', 'resos', or omitted for all")
+
+    background_tasks.add_task(
+        run_aggregation,
+        source=source
+    )
+
+    return {
+        "status": "started",
+        "source": source or "all",
+        "message": f"Aggregation started for {source or 'all sources'}"
+    }
+
+
+@router.post("/aggregate/requeue")
+async def requeue_for_aggregation(
+    background_tasks: BackgroundTasks,
+    source: str = Query(..., description="Source to requeue: newbook or resos"),
+    from_date: Optional[date] = Query(None, description="Start date (optional, defaults to all)"),
+    to_date: Optional[date] = Query(None, description="End date (optional, defaults to all)"),
+    run_aggregation_after: bool = Query(True, description="Automatically run aggregation after queuing"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Re-queue dates from raw data for aggregation.
+
+    Useful when you want to re-aggregate existing data (e.g., after changing
+    mappings or fixing bugs in aggregation logic).
+    """
+    if source not in ['newbook', 'resos']:
+        raise HTTPException(status_code=400, detail="Source must be 'newbook' or 'resos'")
+
+    # Get distinct dates from raw data
+    if source == 'resos':
+        date_query = "SELECT DISTINCT booking_date as date FROM resos_bookings WHERE 1=1"
+    else:  # newbook
+        date_query = """
+            SELECT DISTINCT stay_date as date
+            FROM newbook_booking_nights bn
+            JOIN newbook_bookings b ON bn.booking_id = b.id
+            WHERE 1=1
+        """
+
+    params = {}
+    if from_date:
+        if source == 'resos':
+            date_query += " AND booking_date >= :from_date"
+        else:
+            date_query += " AND stay_date >= :from_date"
+        params["from_date"] = from_date
+
+    if to_date:
+        if source == 'resos':
+            date_query += " AND booking_date <= :to_date"
+        else:
+            date_query += " AND stay_date <= :to_date"
+        params["to_date"] = to_date
+
+    date_query += " ORDER BY date"
+
+    result = await db.execute(text(date_query), params)
+    dates = [row.date for row in result.fetchall()]
+
+    if not dates:
+        return {
+            "status": "no_data",
+            "message": f"No dates found in {source} raw data for the specified range"
+        }
+
+    # Insert dates into queue (delete existing pending entries first, then insert)
+    # Clear any existing pending entries for these dates
+    await db.execute(
+        text("""
+            DELETE FROM aggregation_queue
+            WHERE source = :source
+              AND aggregated_at IS NULL
+        """),
+        {"source": source}
+    )
+
+    # Insert all dates
+    for d in dates:
+        await db.execute(
+            text("""
+                INSERT INTO aggregation_queue (date, source, reason, queued_at)
+                VALUES (:date, :source, 'manual_requeue', NOW())
+            """),
+            {"date": d, "source": source}
+        )
+
+    await db.commit()
+
+    # Optionally trigger aggregation
+    if run_aggregation_after:
+        from jobs.aggregation import run_aggregation as do_aggregation
+        background_tasks.add_task(do_aggregation, source=source)
+
+    return {
+        "status": "queued",
+        "source": source,
+        "dates_queued": len(dates),
+        "date_range": f"{min(dates)} to {max(dates)}",
+        "aggregation_started": run_aggregation_after,
+        "message": f"Queued {len(dates)} dates for {source} aggregation"
+    }
+
+
+@router.get("/aggregate/status")
+async def get_aggregation_status(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get aggregation status summary including pending counts and totals.
+    """
+    # Get pending counts by source
+    pending_query = """
+        SELECT
+            source,
+            COUNT(*) as pending_count,
+            MIN(date) as earliest_date,
+            MAX(date) as latest_date
+        FROM aggregation_queue
+        WHERE aggregated_at IS NULL
+        GROUP BY source
+    """
+    result = await db.execute(text(pending_query))
+    pending_rows = result.fetchall()
+
+    pending_by_source = {
+        row.source: {
+            "count": row.pending_count,
+            "earliest": row.earliest_date,
+            "latest": row.latest_date
+        }
+        for row in pending_rows
+    }
+
+    # Get total pending count
+    total_pending_result = await db.execute(
+        text("SELECT COUNT(*) as total FROM aggregation_queue WHERE aggregated_at IS NULL")
+    )
+    total_pending = total_pending_result.fetchone().total
+
+    # Get aggregated totals
+    occupancy_result = await db.execute(
+        text("SELECT COUNT(*) as count, MIN(date) as earliest, MAX(date) as latest FROM daily_occupancy")
+    )
+    occupancy_row = occupancy_result.fetchone()
+
+    covers_result = await db.execute(
+        text("SELECT COUNT(*) as count, MIN(date) as earliest, MAX(date) as latest FROM daily_covers")
+    )
+    covers_row = covers_result.fetchone()
+
+    # Get last aggregation timestamp (from most recent processed queue entry)
+    last_agg_result = await db.execute(
+        text("SELECT MAX(aggregated_at) as last_run FROM aggregation_queue WHERE aggregated_at IS NOT NULL")
+    )
+    last_agg_row = last_agg_result.fetchone()
+
+    return {
+        "pending": {
+            "total": total_pending,
+            "by_source": pending_by_source
+        },
+        "aggregated": {
+            "daily_occupancy": {
+                "count": occupancy_row.count if occupancy_row else 0,
+                "earliest": occupancy_row.earliest if occupancy_row else None,
+                "latest": occupancy_row.latest if occupancy_row else None
+            },
+            "daily_covers": {
+                "count": covers_row.count if covers_row else 0,
+                "earliest": covers_row.earliest if covers_row else None,
+                "latest": covers_row.latest if covers_row else None
+            }
+        },
+        "last_aggregation": last_agg_row.last_run if last_agg_row else None
+    }
+
+
+@router.get("/aggregate/queue")
+async def get_aggregation_queue(
+    source: Optional[str] = Query(None, description="Filter by source"),
+    pending_only: bool = Query(True, description="Only show pending (un-aggregated) entries"),
+    limit: int = Query(100, description="Max entries to return"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    View the aggregation queue status.
+    """
+    query = """
+        SELECT date, source, reason, booking_id, queued_at, aggregated_at
+        FROM aggregation_queue
+        WHERE 1=1
+    """
+    params = {"limit": limit}
+
+    if source:
+        query += " AND source = :source"
+        params["source"] = source
+
+    if pending_only:
+        query += " AND aggregated_at IS NULL"
+
+    query += " ORDER BY date, source LIMIT :limit"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return {
+        "count": len(rows),
+        "entries": [
+            {
+                "date": row.date,
+                "source": row.source,
+                "reason": row.reason,
+                "booking_id": row.booking_id,
+                "queued_at": row.queued_at,
+                "aggregated_at": row.aggregated_at
+            }
+            for row in rows
+        ]
+    }
+
+
+@router.get("/logs")
+async def get_sync_logs(
+    source: Optional[str] = Query(None, description="Filter by source: newbook, resos"),
+    limit: int = Query(20, description="Number of logs to return"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get recent sync logs for monitoring.
+    """
+    query = """
+        SELECT
+            id,
+            sync_type,
+            source,
+            started_at,
+            completed_at,
+            status,
+            records_fetched,
+            records_created,
+            records_updated,
+            date_from,
+            date_to,
+            error_message,
+            triggered_by
+        FROM sync_log
+    """
+    params = {"limit": limit}
+
+    if source:
+        query += " WHERE source = :source"
+        params["source"] = source
+
+    query += " ORDER BY started_at DESC LIMIT :limit"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "id": row.id,
+            "sync_type": row.sync_type,
+            "source": row.source,
+            "started_at": row.started_at,
+            "completed_at": row.completed_at,
+            "status": row.status,
+            "records_fetched": row.records_fetched,
+            "records_created": row.records_created,
+            "records_updated": row.records_updated,
+            "date_range": f"{row.date_from} to {row.date_to}" if row.date_from else None,
+            "error_message": row.error_message,
+            "triggered_by": row.triggered_by
+        }
+        for row in rows
+    ]
+
+
+# ============================================
+# HISTORICAL BACKFILL ENDPOINTS
+# ============================================
+
+@router.post("/backfill")
+async def start_backfill(
+    request: BackfillRequest,
+    background_tasks: BackgroundTasks,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Start a historical data backfill job.
+
+    Processes data in monthly chunks to avoid API timeouts and rate limits.
+    Progress can be monitored via GET /sync/backfill/status/{job_id}
+    """
+    if request.source not in ['newbook', 'resos', 'all']:
+        raise HTTPException(status_code=400, detail="Source must be 'newbook', 'resos', or 'all'")
+
+    if request.from_date >= request.to_date:
+        raise HTTPException(status_code=400, detail="from_date must be before to_date")
+
+    # Calculate number of chunks
+    total_months = (request.to_date.year - request.from_date.year) * 12 + \
+                   (request.to_date.month - request.from_date.month) + 1
+    chunks_total = (total_months + request.chunk_months - 1) // request.chunk_months
+
+    # Create backfill job record
+    job_id = str(uuid.uuid4())
+    await db.execute(
+        text("""
+            INSERT INTO backfill_jobs (
+                job_id, source, from_date, to_date, chunk_months,
+                status, chunks_total, triggered_by, created_at
+            ) VALUES (
+                :job_id, :source, :from_date, :to_date, :chunk_months,
+                'pending', :chunks_total, :triggered_by, NOW()
+            )
+        """),
+        {
+            "job_id": job_id,
+            "source": request.source,
+            "from_date": request.from_date,
+            "to_date": request.to_date,
+            "chunk_months": request.chunk_months,
+            "chunks_total": chunks_total,
+            "triggered_by": f"user:{current_user['username']}"
+        }
+    )
+    await db.commit()
+
+    # Queue background task
+    background_tasks.add_task(
+        run_backfill_job,
+        job_id=job_id,
+        source=request.source,
+        from_date=request.from_date,
+        to_date=request.to_date,
+        chunk_months=request.chunk_months
+    )
+
+    return {
+        "status": "started",
+        "job_id": job_id,
+        "source": request.source,
+        "from_date": request.from_date,
+        "to_date": request.to_date,
+        "chunk_months": request.chunk_months,
+        "chunks_total": chunks_total,
+        "message": f"Backfill job started. Monitor progress at /sync/backfill/status/{job_id}"
+    }
+
+
+@router.get("/backfill/status/{job_id}")
+async def get_backfill_status(
+    job_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    Get status of a backfill job.
+    """
+    result = await db.execute(
+        text("""
+            SELECT
+                job_id, source, from_date, to_date, chunk_months,
+                status, current_chunk_start, current_chunk_end,
+                chunks_total, chunks_completed, records_total,
+                error_message, started_at, completed_at, triggered_by
+            FROM backfill_jobs
+            WHERE job_id = :job_id
+        """),
+        {"job_id": job_id}
+    )
+    row = result.fetchone()
+
+    if not row:
+        raise HTTPException(status_code=404, detail="Backfill job not found")
+
+    progress_pct = 0
+    if row.chunks_total and row.chunks_total > 0:
+        progress_pct = round((row.chunks_completed / row.chunks_total) * 100, 1)
+
+    return {
+        "job_id": row.job_id,
+        "source": row.source,
+        "date_range": f"{row.from_date} to {row.to_date}",
+        "chunk_months": row.chunk_months,
+        "status": row.status,
+        "progress": {
+            "current_chunk": f"{row.current_chunk_start} to {row.current_chunk_end}" if row.current_chunk_start else None,
+            "chunks_completed": row.chunks_completed,
+            "chunks_total": row.chunks_total,
+            "percent_complete": progress_pct,
+            "records_synced": row.records_total
+        },
+        "error_message": row.error_message,
+        "started_at": row.started_at,
+        "completed_at": row.completed_at,
+        "triggered_by": row.triggered_by
+    }
+
+
+@router.get("/backfill/jobs")
+async def list_backfill_jobs(
+    status: Optional[str] = Query(None, description="Filter by status: pending, running, completed, failed"),
+    limit: int = Query(20, description="Number of jobs to return"),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user)
+):
+    """
+    List recent backfill jobs.
+    """
+    query = """
+        SELECT
+            job_id, source, from_date, to_date, status,
+            chunks_completed, chunks_total, records_total,
+            started_at, completed_at
+        FROM backfill_jobs
+    """
+    params = {"limit": limit}
+
+    if status:
+        query += " WHERE status = :status"
+        params["status"] = status
+
+    query += " ORDER BY created_at DESC LIMIT :limit"
+
+    result = await db.execute(text(query), params)
+    rows = result.fetchall()
+
+    return [
+        {
+            "job_id": row.job_id,
+            "source": row.source,
+            "date_range": f"{row.from_date} to {row.to_date}",
+            "status": row.status,
+            "progress": f"{row.chunks_completed}/{row.chunks_total} chunks",
+            "records_synced": row.records_total,
+            "started_at": row.started_at,
+            "completed_at": row.completed_at
+        }
+        for row in rows
+    ]
+
+
+async def run_backfill_job(
+    job_id: str,
+    source: str,
+    from_date: date,
+    to_date: date,
+    chunk_months: int
+):
+    """
+    Background task to run backfill in chunks.
+
+    For Newbook: Uses modified_since/modified_until to backfill booking data by modification date.
+    For Resos: Uses from_date/to_date to backfill by booking date.
+    """
+    from jobs.data_sync import sync_resos_data
+    from services.newbook_client import NewbookClient
+    import json
+
+    db = SyncSessionLocal()
+
+    try:
+        # Mark job as running
+        db.execute(
+            text("""
+                UPDATE backfill_jobs
+                SET status = 'running', started_at = NOW()
+                WHERE job_id = :job_id
+            """),
+            {"job_id": job_id}
+        )
+        db.commit()
+
+        # Process in chunks
+        current_start = from_date
+        chunks_completed = 0
+        total_records = 0
+
+        while current_start <= to_date:
+            # Calculate chunk end date
+            current_end = current_start + relativedelta(months=chunk_months) - timedelta(days=1)
+            if current_end > to_date:
+                current_end = to_date
+
+            logger.info(f"Backfill {job_id}: Processing {current_start} to {current_end}")
+
+            # Update current chunk in job
+            db.execute(
+                text("""
+                    UPDATE backfill_jobs
+                    SET current_chunk_start = :start, current_chunk_end = :end
+                    WHERE job_id = :job_id
+                """),
+                {"job_id": job_id, "start": current_start, "end": current_end}
+            )
+            db.commit()
+
+            # Sync data for this chunk
+            try:
+                if source in ['newbook', 'all']:
+                    # For backfill, do a full sync (no modified_since filter)
+                    # This pulls all bookings - the sync job handles deduplication via upserts
+                    from jobs.data_sync import sync_newbook_data, sync_newbook_occupancy_report
+                    await sync_newbook_data(
+                        full_sync=True,
+                        triggered_by=f"backfill:{job_id}"
+                    )
+                    # Also backfill occupancy report for this chunk
+                    # This provides available rooms, maintenance, official occupancy figures
+                    await sync_newbook_occupancy_report(
+                        from_date=current_start,
+                        to_date=current_end,
+                        triggered_by=f"backfill:{job_id}"
+                    )
+
+                    # Backfill earned revenue for this chunk (historical dates only)
+                    # This provides official financial figures by GL account
+                    from jobs.data_sync import sync_newbook_earned_revenue
+                    # Only sync earned revenue for historical dates (not future)
+                    earned_rev_end = min(current_end, date.today())
+                    if current_start <= earned_rev_end:
+                        await sync_newbook_earned_revenue(
+                            from_date=current_start,
+                            to_date=earned_rev_end,
+                            triggered_by=f"backfill:{job_id}"
+                        )
+
+                if source in ['resos', 'all']:
+                    await sync_resos_data(current_start, current_end, f"backfill:{job_id}")
+
+            except Exception as chunk_error:
+                logger.error(f"Backfill chunk error: {chunk_error}")
+                # Continue with next chunk instead of failing entire job
+
+            chunks_completed += 1
+
+            # Get records synced for this chunk from sync_log
+            result = db.execute(
+                text("""
+                    SELECT COALESCE(SUM(records_fetched), 0) as total
+                    FROM sync_log
+                    WHERE triggered_by = :triggered_by
+                """),
+                {"triggered_by": f"backfill:{job_id}"}
+            )
+            row = result.fetchone()
+            total_records = row.total if row else 0
+
+            # Update progress
+            db.execute(
+                text("""
+                    UPDATE backfill_jobs
+                    SET chunks_completed = :completed, records_total = :records
+                    WHERE job_id = :job_id
+                """),
+                {"job_id": job_id, "completed": chunks_completed, "records": total_records}
+            )
+            db.commit()
+
+            # Move to next chunk
+            current_start = current_end + timedelta(days=1)
+
+            # For Newbook full sync, we only need to run once (not chunked)
+            if source == 'newbook':
+                break
+
+        # Run aggregation after backfill
+        from jobs.aggregation import run_aggregation
+        await run_aggregation()
+
+        # Mark job as completed
+        db.execute(
+            text("""
+                UPDATE backfill_jobs
+                SET status = 'completed', completed_at = NOW()
+                WHERE job_id = :job_id
+            """),
+            {"job_id": job_id}
+        )
+        db.commit()
+
+        logger.info(f"Backfill {job_id} completed: {total_records} records synced")
+
+    except Exception as e:
+        logger.error(f"Backfill {job_id} failed: {e}")
+        db.execute(
+            text("""
+                UPDATE backfill_jobs
+                SET status = 'failed', error_message = :error, completed_at = NOW()
+                WHERE job_id = :job_id
+            """),
+            {"job_id": job_id, "error": str(e)}
+        )
+        db.commit()
+        raise
+    finally:
+        db.close()

backend/auth.py

@@ -0,0 +1,68 @@
+"""
+Auth middleware — verifies the stack's hnf_session cookie using the shared
+CENTRAL_AUTH_SECRET. Replaces the old per-app JWT/users system.
+"""
+import os
+from fastapi import Depends, HTTPException, Request, status
+from jose import JWTError, jwt
+
+CENTRAL_AUTH_SECRET = os.getenv("CENTRAL_AUTH_SECRET", "")
+JWT_ALGORITHM = "HS256"
+APP_SLUG = os.getenv("APP_SLUG", "forecasting")
+
+
+async def get_current_user(request: Request) -> dict:
+    token = request.cookies.get("hnf_session")
+    if not token:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
+
+    try:
+        payload = jwt.decode(token, CENTRAL_AUTH_SECRET, algorithms=[JWT_ALGORITHM])
+    except JWTError:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session")
+
+    apps = payload.get("apps", [])
+    if APP_SLUG not in apps:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No permission for this app")
+
+    prefix = f"{APP_SLUG}:"
+    raw_caps = payload.get("caps", [])
+    if isinstance(raw_caps, list):
+        caps = [c[len(prefix):] for c in raw_caps if c.startswith(prefix)]
+    else:
+        caps = []
+
+    is_admin = payload.get("is_admin", False)
+
+    return {
+        "id": 0,
+        "username": payload.get("sub", ""),
+        "email": payload.get("sub", ""),
+        "display_name": payload.get("name", ""),
+        "name": payload.get("name", ""),
+        "is_admin": is_admin,
+        "caps": caps,
+        "role": "admin" if is_admin else "user",
+    }
+
+
+def has_cap(user: dict, cap: str) -> bool:
+    return user.get("is_admin", False) or cap in user.get("caps", [])
+
+
+def require_cap(cap: str):
+    async def checker(user: dict = Depends(get_current_user)):
+        if not has_cap(user, cap):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=f"Missing capability: {cap}",
+            )
+        return user
+    return checker
+
+
+# Keep get_admin_user for routes that require admin
+async def get_admin_user(user: dict = Depends(get_current_user)) -> dict:
+    if not user.get("is_admin", False):
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
+    return user

backend/database.py

@@ -0,0 +1,42 @@
+"""
+Database connection and session management
+"""
+import os
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+from sqlalchemy.orm import sessionmaker, declarative_base
+from sqlalchemy import create_engine
+
+DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://forecast:forecast_secret@localhost:5432/forecast")
+
+# Convert to async URL
+ASYNC_DATABASE_URL = DATABASE_URL.replace("postgresql://", "postgresql+asyncpg://")
+
+# Async engine for FastAPI
+async_engine = create_async_engine(ASYNC_DATABASE_URL, echo=False)
+AsyncSessionLocal = sessionmaker(
+    async_engine, class_=AsyncSession, expire_on_commit=False
+)
+
+# Sync engine for scheduler jobs and migrations
+sync_engine = create_engine(DATABASE_URL)
+SyncSessionLocal = sessionmaker(bind=sync_engine)
+
+Base = declarative_base()
+
+
+async def get_db():
+    """Dependency for FastAPI endpoints"""
+    async with AsyncSessionLocal() as session:
+        try:
+            yield session
+        finally:
+            await session.close()
+
+
+def get_sync_db():
+    """Get sync session for scheduler jobs"""
+    db = SyncSessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/jobs/__init__.py

@@ -0,0 +1 @@
+# Scheduled jobs

backend/jobs/accuracy_calc.py

@@ -0,0 +1,182 @@
+"""
+Accuracy calculation job
+Compares forecasts to actuals once dates have passed
+"""
+import logging
+from datetime import date, timedelta
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+async def run_accuracy_calculation():
+    """
+    Calculate forecast accuracy for dates that have passed.
+    Updates actual_vs_forecast table with error metrics.
+    """
+    logger.info("Starting accuracy calculation")
+
+    db = next(iter([SyncSessionLocal()]))
+
+    try:
+        # Process yesterday's actuals
+        calc_date = date.today() - timedelta(days=1)
+
+        # Get all metrics
+        metrics_result = db.execute(
+            text("""
+            SELECT metric_code FROM forecast_metrics WHERE is_active = TRUE
+            """)
+        )
+        metrics = [row.metric_code for row in metrics_result.fetchall()]
+
+        for metric_code in metrics:
+            # Get actual value from daily_metrics
+            actual_result = db.execute(
+                text("""
+                SELECT actual_value FROM daily_metrics
+                WHERE date = :calc_date AND metric_code = :metric_code
+                """),
+                {"calc_date": calc_date, "metric_code": metric_code}
+            )
+            actual_row = actual_result.fetchone()
+            actual_value = actual_row.actual_value if actual_row else None
+
+            if actual_value is None:
+                continue  # Skip if no actual available
+
+            # Get forecasts for this date
+            forecast_result = db.execute(
+                text("""
+                SELECT model_type, predicted_value, lower_bound, upper_bound
+                FROM forecasts
+                WHERE forecast_date = :calc_date AND forecast_type = :metric_code
+                """),
+                {"calc_date": calc_date, "metric_code": metric_code}
+            )
+            forecasts = {row.model_type: row for row in forecast_result.fetchall()}
+
+            prophet_forecast = forecasts.get('prophet')
+            xgboost_forecast = forecasts.get('xgboost')
+            pickup_forecast = forecasts.get('pickup')
+            catboost_forecast = forecasts.get('catboost')
+
+            # Calculate errors
+            def calc_error(forecast_val):
+                if forecast_val is None:
+                    return None, None
+                error = actual_value - forecast_val
+                pct_error = (error / actual_value * 100) if actual_value != 0 else None
+                return error, pct_error
+
+            prophet_error, prophet_pct = calc_error(
+                prophet_forecast.predicted_value if prophet_forecast else None
+            )
+            xgboost_error, xgboost_pct = calc_error(
+                xgboost_forecast.predicted_value if xgboost_forecast else None
+            )
+            pickup_error, pickup_pct = calc_error(
+                pickup_forecast.predicted_value if pickup_forecast else None
+            )
+            catboost_error, catboost_pct = calc_error(
+                catboost_forecast.predicted_value if catboost_forecast else None
+            )
+
+            # Determine best model
+            errors = []
+            if prophet_error is not None:
+                errors.append(('prophet', abs(prophet_error)))
+            if xgboost_error is not None:
+                errors.append(('xgboost', abs(xgboost_error)))
+            if pickup_error is not None:
+                errors.append(('pickup', abs(pickup_error)))
+            if catboost_error is not None:
+                errors.append(('catboost', abs(catboost_error)))
+
+            best_model = min(errors, key=lambda x: x[1])[0] if errors else None
+
+            # Get budget value
+            budget_result = db.execute(
+                text("""
+                SELECT budget_value FROM daily_budgets
+                WHERE date = :calc_date AND budget_type = :metric_code
+                """),
+                {"calc_date": calc_date, "metric_code": metric_code}
+            )
+            budget_row = budget_result.fetchone()
+            budget_value = budget_row.budget_value if budget_row else None
+
+            # Upsert accuracy record
+            db.execute(
+                text("""
+                INSERT INTO actual_vs_forecast (
+                    date, metric_type, actual_value,
+                    prophet_forecast, prophet_lower, prophet_upper,
+                    xgboost_forecast, pickup_forecast,
+                    catboost_forecast,
+                    budget_value,
+                    prophet_error, prophet_pct_error,
+                    xgboost_error, xgboost_pct_error,
+                    pickup_error, pickup_pct_error,
+                    catboost_error, catboost_pct_error,
+                    best_model, calculated_at
+                ) VALUES (
+                    :date, :metric_type, :actual,
+                    :prophet_val, :prophet_lower, :prophet_upper,
+                    :xgboost_val, :pickup_val,
+                    :catboost_val,
+                    :budget,
+                    :prophet_error, :prophet_pct,
+                    :xgboost_error, :xgboost_pct,
+                    :pickup_error, :pickup_pct,
+                    :catboost_error, :catboost_pct,
+                    :best_model, NOW()
+                )
+                ON CONFLICT (date, metric_type) DO UPDATE SET
+                    actual_value = :actual,
+                    prophet_error = :prophet_error,
+                    prophet_pct_error = :prophet_pct,
+                    xgboost_error = :xgboost_error,
+                    xgboost_pct_error = :xgboost_pct,
+                    pickup_error = :pickup_error,
+                    pickup_pct_error = :pickup_pct,
+                    catboost_forecast = :catboost_val,
+                    catboost_error = :catboost_error,
+                    catboost_pct_error = :catboost_pct,
+                    best_model = :best_model,
+                    calculated_at = NOW()
+                """),
+                {
+                    "date": calc_date,
+                    "metric_type": metric_code,
+                    "actual": actual_value,
+                    "prophet_val": prophet_forecast.predicted_value if prophet_forecast else None,
+                    "prophet_lower": prophet_forecast.lower_bound if prophet_forecast else None,
+                    "prophet_upper": prophet_forecast.upper_bound if prophet_forecast else None,
+                    "xgboost_val": xgboost_forecast.predicted_value if xgboost_forecast else None,
+                    "pickup_val": pickup_forecast.predicted_value if pickup_forecast else None,
+                    "catboost_val": catboost_forecast.predicted_value if catboost_forecast else None,
+                    "budget": budget_value,
+                    "prophet_error": prophet_error,
+                    "prophet_pct": prophet_pct,
+                    "xgboost_error": xgboost_error,
+                    "xgboost_pct": xgboost_pct,
+                    "pickup_error": pickup_error,
+                    "pickup_pct": pickup_pct,
+                    "catboost_error": catboost_error,
+                    "catboost_pct": catboost_pct,
+                    "best_model": best_model
+                }
+            )
+
+        db.commit()
+        logger.info(f"Accuracy calculation completed for {calc_date}")
+
+    except Exception as e:
+        logger.error(f"Accuracy calculation failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()

backend/jobs/aggregation.py

@@ -0,0 +1,765 @@
+"""
+Aggregation job - calculates daily summaries from raw booking data
+
+Processes dates from aggregation_queue and updates:
+- daily_occupancy (from newbook_bookings)
+- daily_covers (from resos_bookings)
+"""
+import json
+import logging
+from datetime import date, datetime
+from typing import List, Optional
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+async def run_aggregation(source: Optional[str] = None):
+    """
+    Process pending aggregation queue and update daily summary tables.
+
+    Args:
+        source: Optional filter - 'newbook' or 'resos'. If None, processes both.
+    """
+    logger.info(f"Starting aggregation job (source={source or 'all'})")
+
+    db = next(iter([SyncSessionLocal()]))
+
+    try:
+        # Get pending dates from queue
+        query = """
+            SELECT DISTINCT date, source
+            FROM aggregation_queue
+            WHERE aggregated_at IS NULL
+        """
+
+        if source:
+            query += " AND source = :source"
+
+        query += " ORDER BY date"
+
+        result = db.execute(text(query), {"source": source} if source else {})
+        pending = result.fetchall()
+
+        if not pending:
+            logger.info("No pending dates to aggregate")
+            return
+
+        logger.info(f"Found {len(pending)} date/source combinations to aggregate")
+
+        # Group by source
+        newbook_dates = [row.date for row in pending if row.source == 'newbook']
+        resos_dates = [row.date for row in pending if row.source == 'resos']
+
+        # Process Newbook dates
+        if newbook_dates:
+            await aggregate_newbook_dates(db, newbook_dates)
+
+        # Process Resos dates
+        if resos_dates:
+            await aggregate_resos_dates(db, resos_dates)
+
+        # Populate daily_metrics from aggregated data (for forecasting models)
+        all_dates = list(set(newbook_dates + resos_dates))
+        if all_dates:
+            await populate_daily_metrics(db, all_dates)
+
+        logger.info("Aggregation completed successfully")
+
+    except Exception as e:
+        logger.error(f"Aggregation failed: {e}")
+        raise
+    finally:
+        db.close()
+
+
+async def aggregate_newbook_dates(db, dates: List[date]):
+    """
+    Aggregate newbook_bookings into daily_occupancy for specified dates.
+
+    Room availability is sourced from newbook_occupancy_report table (preferred)
+    which provides accurate available rooms accounting for maintenance/offline rooms.
+    Falls back to system config total_rooms if no occupancy report data exists.
+
+    Revenue metrics:
+    - room_revenue, adr, revpar = NET values (after VAT)
+    - agr = Actual Guest Rate (gross rate guest paid, from calculated_amount)
+    """
+    logger.info(f"Aggregating {len(dates)} Newbook dates")
+
+    # Get fallback total_rooms from system config (used when no occupancy report data)
+    result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'total_rooms'")
+    )
+    row = result.fetchone()
+    fallback_total_rooms = int(row.config_value) if row and row.config_value else 80
+
+    # Get accommodation VAT rate from config (default 20%)
+    result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
+    )
+    row = result.fetchone()
+    accommodation_vat = float(row.config_value) if row and row.config_value else 0.20
+
+    # Statuses that count as "occupied" (case-insensitive check in query)
+    # Includes: Confirmed, Unconfirmed, Arrived, Departed, In-House, etc.
+    # Excludes: Cancelled, No Show, Quote, Waitlist
+    excluded_statuses = "('cancelled', 'no show', 'no_show', 'quote', 'waitlist')"
+
+    # Overflow room category (category_id=5) is used for chargeable no-shows/cancellations
+    # and should be excluded from room night counts
+    overflow_category_id = '5'
+
+    for d in dates:
+        # Get room availability from daily_occupancy (pre-calculated by occupancy report sync)
+        # These values account for maintenance/offline rooms
+        # Revenue comes from booking data, not occupancy report
+        result = db.execute(
+            text("""
+            SELECT
+                total_rooms, available_rooms, maintenance_rooms,
+                newbook_occupied, newbook_occupancy_pct
+            FROM daily_occupancy
+            WHERE date = :date
+            """),
+            {"date": d}
+        )
+        existing = result.fetchone()
+
+        # Use existing availability values if present, else fallback to config
+        if existing and existing.available_rooms and existing.available_rooms > 0:
+            total_rooms = existing.total_rooms
+            available_rooms = existing.available_rooms
+            maintenance_rooms = existing.maintenance_rooms or 0
+            newbook_occupied = existing.newbook_occupied
+            newbook_occupancy_pct = float(existing.newbook_occupancy_pct or 0)
+        else:
+            # No occupancy data yet - fall back to config
+            total_rooms = fallback_total_rooms
+            available_rooms = fallback_total_rooms  # Assume all rooms available
+            maintenance_rooms = 0
+            newbook_occupied = None
+            newbook_occupancy_pct = None
+
+        # Calculate occupancy stats for this date from booking data
+        # A booking is "in house" if: arrival_date <= date < departure_date
+        # AND status is not cancelled/no-show/quote/waitlist
+        # EXCLUDES overflow room category (used for chargeable no-shows)
+        result = db.execute(
+            text(f"""
+            SELECT
+                COUNT(*) as occupied_rooms,
+                COALESCE(SUM(total_guests), 0) as total_guests,
+                COALESCE(SUM(adults), 0) as total_adults,
+                COALESCE(SUM(children), 0) as total_children,
+                COALESCE(SUM(infants), 0) as total_infants
+            FROM newbook_bookings
+            WHERE arrival_date <= :date
+              AND departure_date > :date
+              AND LOWER(status) NOT IN {excluded_statuses}
+              AND (category_id IS NULL OR category_id != :overflow_cat)
+            """),
+            {"date": d, "overflow_cat": overflow_category_id}
+        )
+        stats = result.fetchone()
+
+        # Count arrivals for this date (active bookings only, excluding overflow)
+        result = db.execute(
+            text(f"""
+            SELECT COUNT(*) as arrival_count
+            FROM newbook_bookings
+            WHERE arrival_date = :date
+              AND LOWER(status) NOT IN {excluded_statuses}
+              AND (category_id IS NULL OR category_id != :overflow_cat)
+            """),
+            {"date": d, "overflow_cat": overflow_category_id}
+        )
+        arrivals = result.fetchone()
+
+        # Calculate room revenue, breakfast/dinner allocations from booking_nights
+        # charge_amount = room rate (net of inventory items, but includes VAT)
+        # calculated_amount = gross rate guest paid (for AGR)
+        # GL code matching is done during sync for meal allocations
+        # EXCLUDES overflow category (chargeable no-shows are not actual room stays)
+        result = db.execute(
+            text(f"""
+            SELECT
+                COALESCE(SUM(bn.charge_amount), 0) as charge_amount_total,
+                COALESCE(SUM(bn.calculated_amount), 0) as calculated_amount_total,
+                COALESCE(SUM(CASE WHEN bn.breakfast_gross > 0 THEN 1 ELSE 0 END), 0) as breakfast_qty,
+                COALESCE(SUM(bn.breakfast_net), 0) as breakfast_value,
+                COALESCE(SUM(CASE WHEN bn.dinner_gross > 0 THEN 1 ELSE 0 END), 0) as dinner_qty,
+                COALESCE(SUM(bn.dinner_net), 0) as dinner_value
+            FROM newbook_booking_nights bn
+            JOIN newbook_bookings b ON bn.booking_id = b.id
+            WHERE bn.stay_date = :date
+              AND LOWER(b.status) NOT IN {excluded_statuses}
+              AND (b.category_id IS NULL OR b.category_id != :overflow_cat)
+            """),
+            {"date": d, "overflow_cat": overflow_category_id}
+        )
+        revenue_and_meals = result.fetchone()
+
+        # Revenue breakdown by room category (for revenue_by_room_type JSON)
+        # EXCLUDES overflow category from breakdown
+        result = db.execute(
+            text(f"""
+            SELECT
+                COALESCE(b.category_id, 'unknown') as category_id,
+                COUNT(DISTINCT b.id) as rooms,
+                COALESCE(SUM(bn.charge_amount), 0) as charge_amount,
+                COALESCE(SUM(bn.calculated_amount), 0) as calculated_amount
+            FROM newbook_booking_nights bn
+            JOIN newbook_bookings b ON bn.booking_id = b.id
+            WHERE bn.stay_date = :date
+              AND LOWER(b.status) NOT IN {excluded_statuses}
+              AND (b.category_id IS NULL OR b.category_id != :overflow_cat)
+            GROUP BY b.category_id
+            """),
+            {"date": d, "overflow_cat": overflow_category_id}
+        )
+        revenue_by_category_rows = result.fetchall()
+
+        # Booking movement stats - count by status category
+        result = db.execute(
+            text("""
+            SELECT
+                COUNT(*) FILTER (WHERE LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')) as total_bookings,
+                COUNT(*) FILTER (WHERE LOWER(status) IN ('cancelled')) as cancelled_bookings,
+                COUNT(*) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')) as no_show_bookings
+            FROM newbook_bookings
+            WHERE arrival_date <= :date
+              AND departure_date > :date
+            """),
+            {"date": d}
+        )
+        movement = result.fetchone()
+
+        # Breakdown by room category (keyed by category_id for stability)
+        # EXCLUDES overflow category
+        result = db.execute(
+            text(f"""
+            SELECT
+                COALESCE(category_id, 'unknown') as category_id,
+                COUNT(*) as rooms,
+                COALESCE(SUM(total_guests), 0) as guests,
+                COALESCE(SUM(adults), 0) as adults,
+                COALESCE(SUM(children), 0) as children,
+                COALESCE(SUM(infants), 0) as infants
+            FROM newbook_bookings
+            WHERE arrival_date <= :date
+              AND departure_date > :date
+              AND LOWER(status) NOT IN {excluded_statuses}
+              AND (category_id IS NULL OR category_id != :overflow_cat)
+            GROUP BY category_id
+            """),
+            {"date": d, "overflow_cat": overflow_category_id}
+        )
+        room_type_rows = result.fetchall()
+
+        # Keyed by category_id - use room_categories table to get names in UI
+        by_room_type = {}
+        for row in room_type_rows:
+            by_room_type[row.category_id] = {
+                "rooms": row.rooms,
+                "guests": row.guests,
+                "adults": row.adults,
+                "children": row.children,
+                "infants": row.infants
+            }
+
+        occupied_rooms = stats.occupied_rooms or 0
+        # Use available_rooms (accounts for maintenance) for accurate occupancy %
+        occupancy_pct = (occupied_rooms / available_rooms * 100) if available_rooms > 0 else 0
+
+        # Calculate revenue metrics
+        # charge_amount is the room rate (includes VAT), convert to NET
+        charge_amount_total = float(revenue_and_meals.charge_amount_total or 0)
+        room_revenue = charge_amount_total / (1 + accommodation_vat)  # NET room revenue
+
+        # ADR and RevPAR are NET values
+        # ADR uses occupied rooms, RevPAR uses available rooms
+        adr = (room_revenue / occupied_rooms) if occupied_rooms > 0 else 0
+        revpar = (room_revenue / available_rooms) if available_rooms > 0 else 0
+
+        # AGR (Actual Guest Rate) = gross rate guest paid (from calculated_amount)
+        calculated_amount_total = float(revenue_and_meals.calculated_amount_total or 0)
+        agr = (calculated_amount_total / occupied_rooms) if occupied_rooms > 0 else 0
+
+        # Build revenue_by_room_type JSON with net revenue, ADR, AGR per category
+        revenue_by_room_type = {}
+        for row in revenue_by_category_rows:
+            cat_charge = float(row.charge_amount or 0)
+            cat_calculated = float(row.calculated_amount or 0)
+            cat_rooms = row.rooms or 0
+            cat_revenue_net = cat_charge / (1 + accommodation_vat)
+
+            revenue_by_room_type[row.category_id] = {
+                "rooms": cat_rooms,
+                "revenue_net": round(cat_revenue_net, 2),
+                "adr_net": round(cat_revenue_net / cat_rooms, 2) if cat_rooms > 0 else 0,
+                "agr_total": round(cat_calculated, 2),
+                "agr_avg": round(cat_calculated / cat_rooms, 2) if cat_rooms > 0 else 0
+            }
+
+        # Upsert into daily_occupancy
+        # Revenue comes from booking data (room_revenue, adr, revpar, agr)
+        db.execute(
+            text("""
+            INSERT INTO daily_occupancy (
+                date, total_rooms, available_rooms, maintenance_rooms, occupied_rooms, occupancy_pct,
+                newbook_occupied, newbook_occupancy_pct,
+                total_guests, total_adults, total_children, total_infants,
+                arrival_count, total_bookings, cancelled_bookings, no_show_bookings,
+                room_revenue, adr, revpar, agr,
+                breakfast_allocation_qty, breakfast_allocation_value,
+                dinner_allocation_qty, dinner_allocation_value,
+                by_room_type, revenue_by_room_type, fetched_at
+            ) VALUES (
+                :date, :total_rooms, :available_rooms, :maintenance_rooms, :occupied_rooms, :occupancy_pct,
+                :newbook_occupied, :newbook_occupancy_pct,
+                :total_guests, :total_adults, :total_children, :total_infants,
+                :arrival_count, :total_bookings, :cancelled_bookings, :no_show_bookings,
+                :room_revenue, :adr, :revpar, :agr,
+                :breakfast_qty, :breakfast_value,
+                :dinner_qty, :dinner_value,
+                :by_room_type, :revenue_by_room_type, NOW()
+            )
+            ON CONFLICT (date) DO UPDATE SET
+                total_rooms = :total_rooms,
+                available_rooms = :available_rooms,
+                maintenance_rooms = :maintenance_rooms,
+                occupied_rooms = :occupied_rooms,
+                occupancy_pct = :occupancy_pct,
+                newbook_occupied = :newbook_occupied,
+                newbook_occupancy_pct = :newbook_occupancy_pct,
+                total_guests = :total_guests,
+                total_adults = :total_adults,
+                total_children = :total_children,
+                total_infants = :total_infants,
+                arrival_count = :arrival_count,
+                total_bookings = :total_bookings,
+                cancelled_bookings = :cancelled_bookings,
+                no_show_bookings = :no_show_bookings,
+                room_revenue = :room_revenue,
+                adr = :adr,
+                revpar = :revpar,
+                agr = :agr,
+                breakfast_allocation_qty = :breakfast_qty,
+                breakfast_allocation_value = :breakfast_value,
+                dinner_allocation_qty = :dinner_qty,
+                dinner_allocation_value = :dinner_value,
+                by_room_type = :by_room_type,
+                revenue_by_room_type = :revenue_by_room_type,
+                fetched_at = NOW()
+            """),
+            {
+                "date": d,
+                "total_rooms": total_rooms,
+                "available_rooms": available_rooms,
+                "maintenance_rooms": maintenance_rooms,
+                "occupied_rooms": occupied_rooms,
+                "occupancy_pct": round(occupancy_pct, 2),
+                "newbook_occupied": newbook_occupied,
+                "newbook_occupancy_pct": round(newbook_occupancy_pct, 2) if newbook_occupancy_pct is not None else None,
+                "total_guests": stats.total_guests,
+                "total_adults": stats.total_adults,
+                "total_children": stats.total_children,
+                "total_infants": stats.total_infants,
+                "arrival_count": arrivals.arrival_count or 0,
+                "total_bookings": movement.total_bookings or 0,
+                "cancelled_bookings": movement.cancelled_bookings or 0,
+                "no_show_bookings": movement.no_show_bookings or 0,
+                "room_revenue": round(room_revenue, 2),
+                "adr": round(adr, 2),
+                "revpar": round(revpar, 2),
+                "agr": round(agr, 2),
+                "breakfast_qty": revenue_and_meals.breakfast_qty or 0,
+                "breakfast_value": revenue_and_meals.breakfast_value or 0,
+                "dinner_qty": revenue_and_meals.dinner_qty or 0,
+                "dinner_value": revenue_and_meals.dinner_value or 0,
+                "by_room_type": json.dumps(by_room_type),
+                "revenue_by_room_type": json.dumps(revenue_by_room_type)
+            }
+        )
+
+        # Mark queue entries as processed
+        db.execute(
+            text("""
+            UPDATE aggregation_queue
+            SET aggregated_at = NOW()
+            WHERE date = :date AND source = 'newbook' AND aggregated_at IS NULL
+            """),
+            {"date": d}
+        )
+
+    db.commit()
+    logger.info(f"Aggregated {len(dates)} Newbook dates into daily_occupancy")
+
+
+def load_opening_hours_mappings(db) -> dict:
+    """
+    Load opening hours to period type mappings from resos_opening_hours_mapping table.
+
+    Returns dict: {opening_hour_id: period_type}
+    Where period_type is one of: 'lunch', 'afternoon', 'dinner', 'ignore'
+    """
+    result = db.execute(text("""
+        SELECT opening_hour_id, period_type
+        FROM resos_opening_hours_mapping
+        WHERE period_type != 'ignore'
+    """))
+    mappings = {}
+    for row in result.fetchall():
+        mappings[row.opening_hour_id] = row.period_type
+    return mappings
+
+
+async def aggregate_resos_dates(db, dates: List[date]):
+    """
+    Aggregate resos_bookings into daily_covers for specified dates.
+
+    Uses opening hours mapping to determine service periods (lunch, afternoon, dinner).
+    Falls back to time-based logic if no mappings configured.
+    """
+    logger.info(f"Aggregating {len(dates)} Resos dates")
+
+    # Load opening hours to period type mappings
+    oh_mappings = load_opening_hours_mappings(db)
+    use_oh_mapping = len(oh_mappings) > 0
+    if use_oh_mapping:
+        logger.info(f"Using {len(oh_mappings)} opening hours mappings for period detection")
+    else:
+        logger.info("No opening hours mappings configured, using time-based period detection")
+
+    # Status values that count as "active" (case-insensitive check in query)
+    # Excludes: Cancelled, No Show
+    excluded_statuses = "('cancelled', 'no show', 'no_show')"
+
+    for d in dates:
+        # Get hotel occupancy data for this date (for dining rate calculation)
+        occ_result = db.execute(
+            text("""
+            SELECT total_guests FROM daily_occupancy WHERE date = :date
+            """),
+            {"date": d}
+        )
+        occ_row = occ_result.fetchone()
+        total_hotel_residents = occ_row.total_guests if occ_row and occ_row.total_guests else None
+
+        # Calculate covers by service period
+        # If we have opening hours mappings, aggregate by mapped period_type
+        # Otherwise fall back to simple time-based logic
+        for period in ['lunch', 'afternoon', 'dinner']:
+            if use_oh_mapping:
+                # Get the opening_hour_ids that map to this period
+                period_oh_ids = [oh_id for oh_id, pt in oh_mappings.items() if pt == period]
+
+                if not period_oh_ids:
+                    # No mappings for this period, skip
+                    continue
+
+                # Build SQL placeholders for opening_hour_ids
+                oh_placeholders = ", ".join([f":oh_{i}" for i in range(len(period_oh_ids))])
+                oh_params = {f"oh_{i}": oh_id for i, oh_id in enumerate(period_oh_ids)}
+                oh_params["date"] = d
+
+                period_filter = f"opening_hour_id IN ({oh_placeholders})"
+
+                # Get active booking stats
+                result = db.execute(
+                    text(f"""
+                    SELECT
+                        COUNT(*) as total_bookings,
+                        COALESCE(SUM(covers), 0) as total_covers,
+                        COALESCE(SUM(CASE WHEN is_hotel_guest THEN covers ELSE 0 END), 0) as hotel_guest_covers,
+                        COALESCE(SUM(CASE WHEN NOT is_hotel_guest OR is_hotel_guest IS NULL THEN covers ELSE 0 END), 0) as external_covers,
+                        COALESCE(SUM(CASE WHEN is_dbb THEN covers ELSE 0 END), 0) as dbb_covers,
+                        COALESCE(SUM(CASE WHEN is_package THEN covers ELSE 0 END), 0) as package_covers
+                    FROM resos_bookings
+                    WHERE booking_date = :date
+                      AND {period_filter}
+                      AND LOWER(status) NOT IN {excluded_statuses}
+                    """),
+                    oh_params
+                )
+                stats = result.fetchone()
+
+                # Get cancelled/no-show stats separately
+                result = db.execute(
+                    text(f"""
+                    SELECT
+                        COUNT(*) FILTER (WHERE LOWER(status) = 'cancelled') as cancelled_bookings,
+                        COALESCE(SUM(covers) FILTER (WHERE LOWER(status) = 'cancelled'), 0) as cancelled_covers,
+                        COUNT(*) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')) as no_show_bookings,
+                        COALESCE(SUM(covers) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')), 0) as no_show_covers
+                    FROM resos_bookings
+                    WHERE booking_date = :date
+                      AND {period_filter}
+                    """),
+                    oh_params
+                )
+                movement = result.fetchone()
+
+                # Get source breakdown as JSON
+                result = db.execute(
+                    text(f"""
+                    SELECT
+                        COALESCE(source, 'unknown') as source,
+                        COUNT(*) as bookings,
+                        COALESCE(SUM(covers), 0) as covers
+                    FROM resos_bookings
+                    WHERE booking_date = :date
+                      AND {period_filter}
+                      AND LOWER(status) NOT IN {excluded_statuses}
+                    GROUP BY source
+                    """),
+                    oh_params
+                )
+                source_rows = result.fetchall()
+                by_source = {row.source: {"bookings": row.bookings, "covers": row.covers} for row in source_rows}
+
+            else:
+                # Fallback: time-based logic (skip afternoon if using fallback)
+                if period == 'afternoon':
+                    continue
+
+                if period == 'lunch':
+                    time_filter = "booking_time < '15:00'"
+                else:  # dinner
+                    time_filter = "booking_time >= '15:00'"
+
+                # Get active booking stats
+                result = db.execute(
+                    text(f"""
+                    SELECT
+                        COUNT(*) as total_bookings,
+                        COALESCE(SUM(covers), 0) as total_covers,
+                        COALESCE(SUM(CASE WHEN is_hotel_guest THEN covers ELSE 0 END), 0) as hotel_guest_covers,
+                        COALESCE(SUM(CASE WHEN NOT is_hotel_guest OR is_hotel_guest IS NULL THEN covers ELSE 0 END), 0) as external_covers,
+                        COALESCE(SUM(CASE WHEN is_dbb THEN covers ELSE 0 END), 0) as dbb_covers,
+                        COALESCE(SUM(CASE WHEN is_package THEN covers ELSE 0 END), 0) as package_covers
+                    FROM resos_bookings
+                    WHERE booking_date = :date
+                      AND {time_filter}
+                      AND LOWER(status) NOT IN {excluded_statuses}
+                    """),
+                    {"date": d}
+                )
+                stats = result.fetchone()
+
+                # Get cancelled/no-show stats separately
+                result = db.execute(
+                    text(f"""
+                    SELECT
+                        COUNT(*) FILTER (WHERE LOWER(status) = 'cancelled') as cancelled_bookings,
+                        COALESCE(SUM(covers) FILTER (WHERE LOWER(status) = 'cancelled'), 0) as cancelled_covers,
+                        COUNT(*) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')) as no_show_bookings,
+                        COALESCE(SUM(covers) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')), 0) as no_show_covers
+                    FROM resos_bookings
+                    WHERE booking_date = :date
+                      AND {time_filter}
+                    """),
+                    {"date": d}
+                )
+                movement = result.fetchone()
+
+                # Get source breakdown as JSON
+                result = db.execute(
+                    text(f"""
+                    SELECT
+                        COALESCE(source, 'unknown') as source,
+                        COUNT(*) as bookings,
+                        COALESCE(SUM(covers), 0) as covers
+                    FROM resos_bookings
+                    WHERE booking_date = :date
+                      AND {time_filter}
+                      AND LOWER(status) NOT IN {excluded_statuses}
+                    GROUP BY source
+                    """),
+                    {"date": d}
+                )
+                source_rows = result.fetchall()
+                by_source = {row.source: {"bookings": row.bookings, "covers": row.covers} for row in source_rows}
+
+            total_bookings = stats.total_bookings or 0
+            total_covers = stats.total_covers or 0
+            avg_party_size = (total_covers / total_bookings) if total_bookings > 0 else 0
+            hotel_guest_covers = stats.hotel_guest_covers or 0
+
+            # Calculate hotel guest dining rate (% of hotel residents who dined this period)
+            # This enables forecasting: forecast occupancy → apply dining rate → predict hotel guest covers
+            hotel_guest_dining_rate = None
+            if total_hotel_residents and total_hotel_residents > 0 and hotel_guest_covers > 0:
+                hotel_guest_dining_rate = round((hotel_guest_covers / total_hotel_residents) * 100, 2)
+
+            # Upsert into daily_covers
+            db.execute(
+                text("""
+                INSERT INTO daily_covers (
+                    date, service_period, total_bookings, total_covers, avg_party_size,
+                    hotel_guest_covers, external_covers, dbb_covers, package_covers,
+                    total_hotel_residents, hotel_guest_dining_rate,
+                    cancelled_bookings, cancelled_covers, no_show_bookings, no_show_covers,
+                    by_source, fetched_at
+                ) VALUES (
+                    :date, :service_period, :total_bookings, :total_covers, :avg_party_size,
+                    :hotel_guest_covers, :external_covers, :dbb_covers, :package_covers,
+                    :total_hotel_residents, :hotel_guest_dining_rate,
+                    :cancelled_bookings, :cancelled_covers, :no_show_bookings, :no_show_covers,
+                    :by_source, NOW()
+                )
+                ON CONFLICT (date, service_period) DO UPDATE SET
+                    total_bookings = :total_bookings,
+                    total_covers = :total_covers,
+                    avg_party_size = :avg_party_size,
+                    hotel_guest_covers = :hotel_guest_covers,
+                    external_covers = :external_covers,
+                    dbb_covers = :dbb_covers,
+                    package_covers = :package_covers,
+                    total_hotel_residents = :total_hotel_residents,
+                    hotel_guest_dining_rate = :hotel_guest_dining_rate,
+                    cancelled_bookings = :cancelled_bookings,
+                    cancelled_covers = :cancelled_covers,
+                    no_show_bookings = :no_show_bookings,
+                    no_show_covers = :no_show_covers,
+                    by_source = :by_source,
+                    fetched_at = NOW()
+                """),
+                {
+                    "date": d,
+                    "service_period": period,
+                    "total_bookings": total_bookings,
+                    "total_covers": total_covers,
+                    "avg_party_size": round(avg_party_size, 2),
+                    "hotel_guest_covers": hotel_guest_covers,
+                    "external_covers": stats.external_covers or 0,
+                    "dbb_covers": stats.dbb_covers or 0,
+                    "package_covers": stats.package_covers or 0,
+                    "total_hotel_residents": total_hotel_residents,
+                    "hotel_guest_dining_rate": hotel_guest_dining_rate,
+                    "cancelled_bookings": movement.cancelled_bookings or 0,
+                    "cancelled_covers": movement.cancelled_covers or 0,
+                    "no_show_bookings": movement.no_show_bookings or 0,
+                    "no_show_covers": movement.no_show_covers or 0,
+                    "by_source": json.dumps(by_source)
+                }
+            )
+
+        # Mark queue entries as processed
+        db.execute(
+            text("""
+            UPDATE aggregation_queue
+            SET aggregated_at = NOW()
+            WHERE date = :date AND source = 'resos' AND aggregated_at IS NULL
+            """),
+            {"date": d}
+        )
+
+    db.commit()
+    logger.info(f"Aggregated {len(dates)} Resos dates into daily_covers")
+
+
+async def populate_daily_metrics(db, dates: List[date]):
+    """
+    Populate daily_metrics table from daily_occupancy and daily_covers.
+    This table is the source for forecasting models.
+    """
+    logger.info(f"Populating daily_metrics for {len(dates)} dates")
+
+    for d in dates:
+        # Get daily_occupancy data
+        result = db.execute(
+            text("""
+            SELECT
+                occupied_rooms, total_guests, total_adults, total_children,
+                arrival_count, occupancy_pct, adr, revpar,
+                breakfast_allocation_qty, dinner_allocation_qty,
+                room_revenue, available_rooms
+            FROM daily_occupancy
+            WHERE date = :date
+            """),
+            {"date": d}
+        )
+        occupancy = result.fetchone()
+
+        # Get daily_covers data (lunch and dinner)
+        result = db.execute(
+            text("""
+            SELECT
+                service_period, total_bookings, total_covers, avg_party_size
+            FROM daily_covers
+            WHERE date = :date
+            """),
+            {"date": d}
+        )
+        covers_rows = result.fetchall()
+
+        # Build covers data by period
+        covers_data = {}
+        for row in covers_rows:
+            covers_data[row.service_period] = {
+                "bookings": row.total_bookings,
+                "covers": row.total_covers,
+                "party_size": float(row.avg_party_size or 0)
+            }
+
+        # Define metrics to populate
+        metrics_to_insert = []
+
+        if occupancy:
+            # Hotel metrics
+            metrics_to_insert.extend([
+                ("hotel_room_nights", occupancy.occupied_rooms, "newbook"),
+                ("hotel_occupancy_pct", float(occupancy.occupancy_pct or 0), "newbook"),
+                ("hotel_guests", occupancy.total_guests, "newbook"),
+                ("hotel_arrivals", occupancy.arrival_count, "newbook"),
+                ("hotel_adr", float(occupancy.adr or 0), "newbook"),
+                ("hotel_revpar", float(occupancy.revpar or 0), "newbook"),
+                ("hotel_breakfast_qty", occupancy.breakfast_allocation_qty, "newbook"),
+                ("hotel_dinner_qty", occupancy.dinner_allocation_qty, "newbook"),
+                ("revenue_rooms", float(occupancy.room_revenue or 0), "newbook"),
+            ])
+
+        # Restaurant metrics - lunch
+        if "lunch" in covers_data:
+            lunch = covers_data["lunch"]
+            metrics_to_insert.extend([
+                ("resos_lunch_bookings", lunch["bookings"], "resos"),
+                ("resos_lunch_covers", lunch["covers"], "resos"),
+                ("resos_lunch_party_size", lunch["party_size"], "resos"),
+            ])
+
+        # Restaurant metrics - dinner
+        if "dinner" in covers_data:
+            dinner = covers_data["dinner"]
+            metrics_to_insert.extend([
+                ("resos_dinner_bookings", dinner["bookings"], "resos"),
+                ("resos_dinner_covers", dinner["covers"], "resos"),
+                ("resos_dinner_party_size", dinner["party_size"], "resos"),
+            ])
+
+        # Insert/update all metrics
+        for metric_code, actual_value, source in metrics_to_insert:
+            if actual_value is not None:
+                db.execute(
+                    text("""
+                    INSERT INTO daily_metrics (date, metric_code, actual_value, source, calculated_at)
+                    VALUES (:date, :metric_code, :actual_value, :source, NOW())
+                    ON CONFLICT (date, metric_code) DO UPDATE SET
+                        actual_value = :actual_value,
+                        source = :source,
+                        calculated_at = NOW()
+                    """),
+                    {
+                        "date": d,
+                        "metric_code": metric_code,
+                        "actual_value": actual_value,
+                        "source": source
+                    }
+                )
+
+    db.commit()
+    logger.info(f"Populated daily_metrics for {len(dates)} dates")

backend/jobs/ai_insights.py

@@ -0,0 +1,456 @@
+"""
+AI Daily Insights Generation Job
+
+Gathers Pickup-V2 forecast data, booking pace, competitor rates, and rate parity
+information, then sends a compact prompt to Anthropic's Haiku model to generate
+a daily briefing for hotel revenue staff.
+
+Schedule: Daily at 7:15 AM (after all forecasts and accuracy calc complete)
+Cost: ~$0.05/month at 1 run/day with Haiku
+"""
+import json
+import logging
+from datetime import date, timedelta, datetime, timezone
+from typing import Dict, List, Any, Optional
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from database import AsyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MODEL = "claude-haiku-4-5-20251001"
+DEFAULT_DAILY_TOKEN_BUDGET = 5000
+MAX_OUTPUT_TOKENS = 400
+
+
+async def get_config(db: AsyncSession) -> Dict[str, str]:
+    """Get AI insights config from system_config."""
+    result = await db.execute(
+        text("""
+            SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted
+            FROM system_config
+            WHERE config_key LIKE 'ai_insights_%'
+        """)
+    )
+    config = {}
+    for row in result.fetchall():
+        value = row.config_value
+        if row.is_encrypted and value:
+            import base64
+            try:
+                value = base64.b64decode(value.encode()).decode()
+            except Exception:
+                pass
+        config[row.config_key] = value
+    return config
+
+
+async def check_daily_budget(db: AsyncSession, budget: int) -> tuple[bool, int]:
+    """Check if we're within the daily token budget. Returns (within_budget, tokens_used_today)."""
+    result = await db.execute(
+        text("""
+            SELECT COALESCE(SUM(input_tokens + output_tokens), 0) as total
+            FROM ai_insights
+            WHERE generated_at >= CURRENT_DATE
+        """)
+    )
+    row = result.fetchone()
+    used = int(row.total) if row else 0
+    return used < budget, used
+
+
+async def gather_occupancy_data(db: AsyncSession, days: int = 14) -> List[Dict]:
+    """Gather Pickup-V2 occupancy forecast data for the next N days."""
+    from services.forecasting.pickup_v2_model import run_pickup_v2_forecast
+
+    today = date.today()
+    end = today + timedelta(days=days - 1)
+
+    try:
+        forecasts = await run_pickup_v2_forecast(
+            db, 'hotel_occupancy_pct', today, end, include_details=False
+        )
+        return forecasts
+    except Exception as e:
+        logger.warning(f"Failed to gather occupancy data: {e}")
+        return []
+
+
+async def gather_revenue_data(db: AsyncSession, days: int = 14) -> List[Dict]:
+    """Gather Pickup-V2 revenue forecast data for the next N days."""
+    from services.forecasting.pickup_v2_model import run_pickup_v2_forecast
+
+    today = date.today()
+    end = today + timedelta(days=days - 1)
+
+    try:
+        forecasts = await run_pickup_v2_forecast(
+            db, 'net_accom', today, end, include_details=False
+        )
+        return forecasts
+    except Exception as e:
+        logger.warning(f"Failed to gather revenue data: {e}")
+        return []
+
+
+async def gather_budget_data(db: AsyncSession, days: int = 14) -> Dict[str, float]:
+    """Gather budget values for forecast comparison."""
+    today = date.today()
+    end = today + timedelta(days=days - 1)
+
+    result = await db.execute(
+        text("""
+            SELECT date, budget_type, budget_value
+            FROM daily_budgets
+            WHERE date BETWEEN :start AND :end
+            AND budget_type IN ('net_accom', 'occupancy')
+        """),
+        {"start": today, "end": end}
+    )
+
+    budgets = {}
+    for row in result.fetchall():
+        key = f"{row.date}_{row.budget_type}"
+        budgets[key] = float(row.budget_value) if row.budget_value else None
+    return budgets
+
+
+async def gather_competitor_data(db: AsyncSession, days: int = 14) -> Dict[str, Any]:
+    """Gather competitor rate data from Booking.com scraper."""
+    today = date.today()
+    end = today + timedelta(days=days - 1)
+
+    # Own hotel rate on Booking.com
+    own_result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (r.rate_date)
+                r.rate_date,
+                r.rate_gross as booking_rate,
+                r.availability_status
+            FROM booking_com_rates r
+            JOIN booking_com_hotels h ON r.hotel_id = h.id
+            WHERE h.tier = 'own'
+              AND r.rate_date BETWEEN :start AND :end
+            ORDER BY r.rate_date, r.scraped_at DESC
+        """),
+        {"start": today, "end": end}
+    )
+    own_rates = {str(row.rate_date): {
+        'rate': float(row.booking_rate) if row.booking_rate else None,
+        'status': row.availability_status
+    } for row in own_result.fetchall()}
+
+    # Cheapest competitor rate per date
+    comp_result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (r.rate_date)
+                r.rate_date,
+                r.rate_gross as comp_rate,
+                h.name as comp_name
+            FROM booking_com_rates r
+            JOIN booking_com_hotels h ON r.hotel_id = h.id
+            WHERE h.tier IN ('primary', 'secondary')
+              AND r.rate_date BETWEEN :start AND :end
+              AND r.availability_status = 'available'
+              AND r.rate_gross IS NOT NULL
+            ORDER BY r.rate_date, r.rate_gross ASC
+        """),
+        {"start": today, "end": end}
+    )
+    comp_rates = {str(row.rate_date): {
+        'rate': float(row.comp_rate),
+        'name': row.comp_name
+    } for row in comp_result.fetchall()}
+
+    # Own rack rate from Newbook
+    rack_result = await db.execute(
+        text("""
+            SELECT DISTINCT ON (rate_date)
+                rate_date,
+                rate_gross
+            FROM newbook_current_rates
+            WHERE rate_date BETWEEN :start AND :end
+            ORDER BY rate_date, valid_from DESC
+        """),
+        {"start": today, "end": end}
+    )
+    rack_rates = {str(row.rate_date): float(row.rate_gross) if row.rate_gross else None
+                  for row in rack_result.fetchall()}
+
+    return {
+        'own_booking': own_rates,
+        'competitors': comp_rates,
+        'rack': rack_rates
+    }
+
+
+def build_prompt(
+    occupancy: List[Dict],
+    revenue: List[Dict],
+    budgets: Dict[str, float],
+    competitor: Dict[str, Any]
+) -> tuple[str, str]:
+    """Build system and user prompts from gathered data. Returns (system_msg, user_msg)."""
+
+    system_msg = (
+        "You are an AI assistant for a hotel revenue manager. Analyze the data below and provide "
+        "a concise daily briefing (3-5 bullet points). Focus on: occupancy trends, pace vs prior "
+        "year, pricing opportunities, rate parity issues, and anything unusual requiring attention. "
+        "Be specific with numbers and dates. Keep it actionable — no fluff or generic advice."
+    )
+
+    lines = []
+
+    # Occupancy section
+    if occupancy:
+        lines.append("## Occupancy Forecast - Pickup-V2 (next 14 days)")
+        lines.append("Date | DoW | OTB | Forecast | PY Final | Pace vs LY | Budget")
+        for fc in occupancy:
+            d = fc.get('date', '')
+            dow = fc.get('day_of_week', '')
+            otb = fc.get('current_otb')
+            forecast = fc.get('forecast')
+            py_final = fc.get('prior_year_final')
+            pace = fc.get('pace_vs_prior_pct')
+            budget_key = f"{d}_occupancy"
+            budget_val = budgets.get(budget_key)
+
+            otb_str = f"{otb:.0f}%" if otb is not None else "-"
+            fc_str = f"{forecast:.0f}%" if forecast is not None else "-"
+            py_str = f"{py_final:.0f}%" if py_final is not None else "-"
+            pace_str = f"{pace:+.0f}%" if pace is not None else "-"
+            bud_str = f"{budget_val:.0f}%" if budget_val is not None else "-"
+
+            lines.append(f"{d} | {dow} | {otb_str} | {fc_str} | {py_str} | {pace_str} | {bud_str}")
+        lines.append("")
+
+    # Revenue summary
+    if revenue:
+        total_forecast = sum(fc.get('forecast', 0) or 0 for fc in revenue)
+        total_otb = sum(fc.get('current_otb_rev', 0) or 0 for fc in revenue)
+        total_py = sum(fc.get('prior_year_final_rev', 0) or 0 for fc in revenue)
+        opportunity_days = sum(1 for fc in revenue if fc.get('has_pricing_opportunity'))
+        total_lost = sum(fc.get('lost_potential', 0) or 0 for fc in revenue)
+
+        lines.append("## Revenue Signals")
+        lines.append(f"14-day forecast: ${total_forecast:,.0f} | OTB: ${total_otb:,.0f} | PY: ${total_py:,.0f}")
+        if opportunity_days > 0:
+            lines.append(f"Pricing opportunity days: {opportunity_days} | Total lost potential: ${total_lost:,.0f}")
+        lines.append("")
+
+    # Competitor rates section
+    own_booking = competitor.get('own_booking', {})
+    comp_rates = competitor.get('competitors', {})
+    rack_rates = competitor.get('rack', {})
+
+    if own_booking or comp_rates:
+        lines.append("## Competitor Rates (next 14 days)")
+        lines.append("Date | Own Rack | Own B.com | Cheapest Competitor | Competitor Name")
+
+        all_dates = sorted(set(list(own_booking.keys()) + list(comp_rates.keys()) + list(rack_rates.keys())))
+        for d in all_dates:
+            rack = rack_rates.get(d)
+            own = own_booking.get(d, {})
+            comp = comp_rates.get(d, {})
+
+            rack_str = f"${rack:.0f}" if rack else "-"
+            own_str = f"${own['rate']:.0f}" if own.get('rate') else "-"
+            comp_str = f"${comp['rate']:.0f}" if comp.get('rate') else "-"
+            comp_name = comp.get('name', '-')
+
+            note = ""
+            if own.get('rate') and comp.get('rate') and own['rate'] < comp['rate']:
+                note = " <- cheapest on B.com"
+
+            lines.append(f"{d} | {rack_str} | {own_str} | {comp_str} | {comp_name}{note}")
+        lines.append("")
+
+    # Rate parity flags
+    parity_flags = []
+    for d in sorted(rack_rates.keys()):
+        rack = rack_rates.get(d)
+        own = own_booking.get(d, {})
+        if rack and own.get('rate') and rack > 0:
+            delta_pct = ((own['rate'] - rack) / rack) * 100
+            if abs(delta_pct) > 5:
+                parity_flags.append(f"{d}: Rack ${rack:.0f} vs B.com ${own['rate']:.0f} ({delta_pct:+.1f}%)")
+
+    if parity_flags:
+        lines.append("## Rate Parity Flags (own rack vs own Booking.com, >5% delta)")
+        for flag in parity_flags:
+            lines.append(flag)
+        lines.append("")
+
+    user_msg = "\n".join(lines)
+    return system_msg, user_msg
+
+
+async def call_llm(api_key: str, system_msg: str, user_msg: str, model: str) -> Dict[str, Any]:
+    """Call Anthropic API and return response with token usage."""
+    import anthropic
+
+    client = anthropic.AsyncAnthropic(api_key=api_key)
+
+    try:
+        response = await client.messages.create(
+            model=model,
+            max_tokens=MAX_OUTPUT_TOKENS,
+            temperature=0.2,
+            system=system_msg,
+            messages=[{"role": "user", "content": user_msg}]
+        )
+
+        content = response.content[0].text if response.content else ""
+        return {
+            "content": content,
+            "input_tokens": response.usage.input_tokens,
+            "output_tokens": response.usage.output_tokens,
+            "model": model,
+        }
+    finally:
+        await client.close()
+
+
+async def save_insight(
+    db: AsyncSession,
+    content: str,
+    model: str,
+    input_tokens: int,
+    output_tokens: int,
+    data_snapshot: Dict,
+    triggered_by: str = "scheduler"
+):
+    """Save generated insight to database."""
+    await db.execute(
+        text("""
+            INSERT INTO ai_insights
+            (content, model, input_tokens, output_tokens, data_snapshot, triggered_by)
+            VALUES (:content, :model, :input_tokens, :output_tokens,
+                    CAST(:data_snapshot AS jsonb), :triggered_by)
+        """),
+        {
+            "content": content,
+            "model": model,
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "data_snapshot": json.dumps(data_snapshot),
+            "triggered_by": triggered_by,
+        }
+    )
+    await db.commit()
+
+
+async def cleanup_old_insights(db: AsyncSession, keep_days: int = 90):
+    """Remove insights older than keep_days."""
+    cutoff = datetime.now(timezone.utc) - timedelta(days=keep_days)
+    await db.execute(
+        text("DELETE FROM ai_insights WHERE generated_at < :cutoff"),
+        {"cutoff": cutoff}
+    )
+    await db.commit()
+
+
+async def generate_insight(db: AsyncSession, triggered_by: str = "scheduler") -> Dict[str, Any]:
+    """
+    Core insight generation logic. Used by both scheduler and manual trigger.
+    Returns result dict with success/error status.
+    """
+    config = await get_config(db)
+
+    # Check enabled
+    if config.get('ai_insights_enabled', 'false').lower() not in ('true', '1', 'yes'):
+        return {"success": False, "error": "AI insights disabled"}
+
+    # Check API key
+    api_key = config.get('ai_insights_api_key')
+    if not api_key:
+        return {"success": False, "error": "No API key configured"}
+
+    model = config.get('ai_insights_model', DEFAULT_MODEL)
+    budget = int(config.get('ai_insights_daily_token_budget', str(DEFAULT_DAILY_TOKEN_BUDGET)))
+
+    # Check daily budget
+    within_budget, used_today = await check_daily_budget(db, budget)
+    if not within_budget:
+        return {
+            "success": False,
+            "error": f"Daily token budget exceeded ({used_today}/{budget} tokens used today)"
+        }
+
+    # Gather data
+    logger.info("Gathering data for AI insight...")
+    occupancy = await gather_occupancy_data(db)
+    revenue = await gather_revenue_data(db)
+    budgets = await gather_budget_data(db)
+    competitor = await gather_competitor_data(db)
+
+    if not occupancy and not revenue:
+        return {"success": False, "error": "No forecast data available"}
+
+    # Build prompt
+    system_msg, user_msg = build_prompt(occupancy, revenue, budgets, competitor)
+
+    # Store data snapshot for debugging
+    data_snapshot = {
+        "occupancy_days": len(occupancy),
+        "revenue_days": len(revenue),
+        "competitor_dates": len(competitor.get('own_booking', {})),
+        "parity_flags": sum(1 for d in competitor.get('rack', {})
+                           if competitor.get('own_booking', {}).get(d, {}).get('rate')
+                           and competitor['rack'].get(d)
+                           and abs((competitor['own_booking'][d]['rate'] - competitor['rack'][d]) / competitor['rack'][d] * 100) > 5),
+        "prompt_preview": user_msg[:500],
+    }
+
+    # Call LLM
+    logger.info(f"Calling {model} for AI insight...")
+    try:
+        result = await call_llm(api_key, system_msg, user_msg, model)
+    except Exception as e:
+        logger.error(f"LLM call failed: {e}")
+        return {"success": False, "error": f"LLM call failed: {str(e)}"}
+
+    # Save
+    await save_insight(
+        db,
+        content=result["content"],
+        model=result["model"],
+        input_tokens=result["input_tokens"],
+        output_tokens=result["output_tokens"],
+        data_snapshot=data_snapshot,
+        triggered_by=triggered_by,
+    )
+
+    logger.info(
+        f"AI insight generated: {result['input_tokens']} input, "
+        f"{result['output_tokens']} output tokens ({triggered_by})"
+    )
+
+    # Cleanup old insights
+    try:
+        await cleanup_old_insights(db)
+    except Exception as e:
+        logger.warning(f"Old insight cleanup failed: {e}")
+
+    return {
+        "success": True,
+        "content": result["content"],
+        "input_tokens": result["input_tokens"],
+        "output_tokens": result["output_tokens"],
+        "model": result["model"],
+    }
+
+
+async def run_ai_insights_generation():
+    """Scheduled job entry point."""
+    async with AsyncSessionLocal() as db:
+        try:
+            result = await generate_insight(db, triggered_by="scheduler")
+            if result.get("success"):
+                logger.info("Scheduled AI insight generation completed")
+            else:
+                logger.info(f"Scheduled AI insight skipped: {result.get('error')}")
+        except Exception as e:
+            logger.error(f"Scheduled AI insight generation failed: {e}", exc_info=True)

backend/jobs/bookings_aggregation.py

@@ -0,0 +1,847 @@
+"""
+Bookings aggregation job - aggregates newbook_bookings_data into:
+- newbook_bookings_stats: daily aggregated stats with JSONB category breakdowns
+- newbook_booking_pace: lead-time snapshots for forecasting pickup patterns
+
+Triggered automatically after bookings sync completes.
+"""
+import json
+import logging
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from typing import List, Set, Dict, Any, Optional
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+# Valid booking statuses for aggregation
+VALID_STATUSES = ('Unconfirmed', 'Confirmed', 'Arrived', 'Departed')
+
+# All tracked pace intervals
+PACE_INTERVALS = [
+    # Monthly (months 7-12)
+    365, 330, 300, 270, 240, 210,
+    # Weekly (weeks 5-25)
+    177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
+    107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
+    # Daily (days 0-30)
+    30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
+    20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
+    10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+]
+
+
+def get_config_value(db, key: str) -> Optional[str]:
+    """Get a configuration value from system_config table."""
+    result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = :key"),
+        {"key": key}
+    )
+    row = result.fetchone()
+    return row.config_value if row else None
+
+
+async def run_bookings_aggregation(triggered_by: str = "manual"):
+    """
+    Aggregate bookings into newbook_bookings_stats.
+
+    Flow:
+    1. Find bookings changed since last_bookings_aggregation_at
+    2. Calculate affected dates (arrival_date <= date < departure_date)
+    3. Reaggregate each affected date
+    4. Update booking pace table
+    5. Update last_bookings_aggregation_at
+    """
+    logger.info(f"Starting bookings aggregation (triggered_by={triggered_by})")
+
+    db = next(iter([SyncSessionLocal()]))
+
+    try:
+        # Get last aggregation timestamp
+        last_aggregation = get_config_value(db, 'last_bookings_aggregation_at')
+        if last_aggregation:
+            try:
+                last_ts = datetime.fromisoformat(last_aggregation)
+            except ValueError:
+                last_ts = datetime.min
+        else:
+            last_ts = datetime.min
+
+        logger.info(f"Last aggregation: {last_ts}")
+
+        # Find bookings changed since last aggregation
+        result = db.execute(
+            text("""
+                SELECT newbook_id, arrival_date, departure_date
+                FROM newbook_bookings_data
+                WHERE fetched_at > :last_ts
+            """),
+            {"last_ts": last_ts}
+        )
+        changed_bookings = result.fetchall()
+
+        if not changed_bookings:
+            logger.info("No changed bookings to aggregate")
+            # Still update pace table
+            await update_booking_pace(db)
+            db.commit()
+            return
+
+        logger.info(f"Found {len(changed_bookings)} changed bookings")
+
+        # Calculate affected dates
+        affected_dates: Set[date] = set()
+        for booking in changed_bookings:
+            if booking.arrival_date and booking.departure_date:
+                current = booking.arrival_date
+                # < not <= (departure is checkout day, guest not staying that night)
+                while current < booking.departure_date:
+                    affected_dates.add(current)
+                    current += timedelta(days=1)
+
+        logger.info(f"Reaggregating {len(affected_dates)} affected dates")
+
+        # Get accommodation VAT rate
+        vat_rate_str = get_config_value(db, 'accommodation_vat_rate')
+        vat_rate = Decimal(vat_rate_str) if vat_rate_str else Decimal('0.20')
+
+        # Aggregate each affected date
+        for target_date in sorted(affected_dates):
+            await aggregate_date(db, target_date, vat_rate)
+
+        # Fill any dates with occupancy data but no bookings (e.g., closed periods)
+        await fill_occupancy_only_dates(db, vat_rate)
+
+        # Update booking pace table
+        await update_booking_pace(db)
+
+        # Update last aggregation timestamp
+        db.execute(
+            text("""
+                INSERT INTO system_config (config_key, config_value, updated_at)
+                VALUES ('last_bookings_aggregation_at', :now, NOW())
+                ON CONFLICT (config_key) DO UPDATE SET
+                    config_value = :now,
+                    updated_at = NOW()
+            """),
+            {"now": datetime.now().isoformat()}
+        )
+
+        db.commit()
+        logger.info(f"Bookings aggregation completed: {len(affected_dates)} dates processed")
+
+    except Exception as e:
+        logger.error(f"Bookings aggregation failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+async def aggregate_date(db, target_date: date, vat_rate: Decimal):
+    """
+    Aggregate all bookings for a specific date into newbook_bookings_stats.
+
+    Includes room availability from newbook_occupancy_report_data and
+    booking stats from newbook_bookings_data.
+    """
+    # Step 1: Get room availability from occupancy report (included categories only)
+    result = db.execute(
+        text("""
+            SELECT
+                o.category_id,
+                COALESCE(o.available, 0) as available,
+                COALESCE(o.maintenance, 0) as maintenance
+            FROM newbook_occupancy_report_data o
+            JOIN newbook_room_categories c ON o.category_id = c.site_id
+            WHERE o.date = :target_date
+            AND c.is_included = true
+        """),
+        {"target_date": target_date}
+    )
+    occupancy_rows = result.fetchall()
+
+    # Build availability by category
+    availability_by_category: Dict[str, Dict[str, Any]] = {}
+    rooms_count = 0
+    maintenance_count = 0
+
+    for row in occupancy_rows:
+        cat_id = row.category_id
+        available = row.available or 0
+        maintenance = row.maintenance or 0
+        bookable = available - maintenance
+
+        rooms_count += available
+        maintenance_count += maintenance
+
+        availability_by_category[cat_id] = {
+            "rooms_count": available,
+            "maintenance_count": maintenance,
+            "bookable_count": bookable,
+            "booking_count": 0,
+            "total_occupancy_pct": None,
+            "bookable_occupancy_pct": None
+        }
+
+    bookable_count = rooms_count - maintenance_count
+
+    # Fallback: If no occupancy data (bookable_count=0), use last known bookable_count
+    # This prevents division-by-zero issues in forecast models when occupancy report is missing
+    if bookable_count <= 0:
+        fallback_result = db.execute(
+            text("""
+                SELECT bookable_count
+                FROM newbook_bookings_stats
+                WHERE bookable_count > 5 AND date < :target_date
+                ORDER BY date DESC
+                LIMIT 1
+            """),
+            {"target_date": target_date}
+        )
+        fallback_row = fallback_result.fetchone()
+        if fallback_row and fallback_row.bookable_count:
+            bookable_count = fallback_row.bookable_count
+            rooms_count = bookable_count  # Assume same for rooms_count
+            logger.info(f"Using fallback bookable_count={bookable_count} for {target_date}")
+
+    # Step 2: Get booking stats (bookings staying this night)
+    # A booking is "in house" if: arrival_date <= date < departure_date
+    # Only counts bookings for categories marked as is_included=true in settings
+    result = db.execute(
+        text("""
+            SELECT
+                b.newbook_id,
+                b.category_id,
+                COALESCE(b.adults, 0) + COALESCE(b.children, 0) + COALESCE(b.infants, 0) as guests,
+                COALESCE(b.adults, 0) as adults,
+                COALESCE(b.children, 0) as children,
+                COALESCE(b.infants, 0) as infants,
+                b.raw_json
+            FROM newbook_bookings_data b
+            JOIN newbook_room_categories c ON b.category_id = c.site_id
+            WHERE b.arrival_date <= :target_date
+            AND b.departure_date > :target_date
+            AND b.status IN :valid_statuses
+            AND c.is_included = true
+        """),
+        {"target_date": target_date, "valid_statuses": VALID_STATUSES}
+    )
+    bookings = result.fetchall()
+
+    # Aggregate bookings
+    booking_count = 0
+    guests_count = 0
+    adults_count = 0
+    children_count = 0
+    infants_count = 0
+    guest_rate_total = Decimal('0')
+    net_booking_rev_total = Decimal('0')
+
+    occupancy_by_category: Dict[str, Dict[str, Any]] = {}
+    revenue_by_category: Dict[str, Dict[str, Any]] = {}
+    rate_stats_by_category: Dict[str, Dict[str, Any]] = {}  # Pickup-V2: min/max/adr per category
+
+    for booking in bookings:
+        booking_count += 1
+        guests_count += booking.guests or 0
+        adults_count += booking.adults or 0
+        children_count += booking.children or 0
+        infants_count += booking.infants or 0
+
+        cat_id = booking.category_id or 'unknown'
+
+        # Initialize category dicts if needed
+        if cat_id not in occupancy_by_category:
+            occupancy_by_category[cat_id] = {
+                "booking_count": 0,
+                "guests": 0,
+                "adults": 0,
+                "children": 0,
+                "infants": 0
+            }
+        if cat_id not in revenue_by_category:
+            revenue_by_category[cat_id] = {
+                "guest_rate_total": Decimal('0'),
+                "net_booking_rev_total": Decimal('0')
+            }
+        if cat_id not in rate_stats_by_category:
+            rate_stats_by_category[cat_id] = {
+                "rates": [],  # Collect all net rates for min/max/adr calculation
+                "rooms": 0
+            }
+
+        # Update occupancy by category
+        occupancy_by_category[cat_id]["booking_count"] += 1
+        occupancy_by_category[cat_id]["guests"] += booking.guests or 0
+        occupancy_by_category[cat_id]["adults"] += booking.adults or 0
+        occupancy_by_category[cat_id]["children"] += booking.children or 0
+        occupancy_by_category[cat_id]["infants"] += booking.infants or 0
+
+        # Update availability by category booking count
+        if cat_id in availability_by_category:
+            availability_by_category[cat_id]["booking_count"] += 1
+
+        # Get revenue from tariffs_quoted for this date
+        calculated_amount, net_amount = get_rate_for_date(
+            booking.raw_json, target_date, vat_rate
+        )
+        guest_rate_total += calculated_amount
+        net_booking_rev_total += net_amount
+
+        revenue_by_category[cat_id]["guest_rate_total"] += calculated_amount
+        revenue_by_category[cat_id]["net_booking_rev_total"] += net_amount
+
+        # Pickup-V2: Collect net rate for rate stats (only if rate > 0)
+        if net_amount > 0:
+            rate_stats_by_category[cat_id]["rates"].append(float(net_amount))
+            rate_stats_by_category[cat_id]["rooms"] += 1
+
+    # Calculate occupancy percentages
+    total_occupancy_pct = None
+    bookable_occupancy_pct = None
+
+    if rooms_count > 0:
+        total_occupancy_pct = round(float(booking_count) / rooms_count * 100, 2)
+    if bookable_count > 0:
+        bookable_occupancy_pct = round(float(booking_count) / bookable_count * 100, 2)
+
+    # Calculate per-category occupancy percentages
+    for cat_id, avail in availability_by_category.items():
+        cat_bookings = avail["booking_count"]
+        cat_rooms = avail["rooms_count"]
+        cat_bookable = avail["bookable_count"]
+
+        if cat_rooms > 0:
+            avail["total_occupancy_pct"] = round(float(cat_bookings) / cat_rooms * 100, 2)
+        if cat_bookable > 0:
+            avail["bookable_occupancy_pct"] = round(float(cat_bookings) / cat_bookable * 100, 2)
+
+    # Convert Decimal to float for JSON serialization
+    def decimal_to_float(d: Dict) -> Dict:
+        return {
+            k: (float(v) if isinstance(v, Decimal) else v)
+            for k, v in d.items()
+        }
+
+    # Pickup-V2: Calculate min/max/adr from collected rates
+    rate_stats_final: Dict[str, Dict[str, Any]] = {}
+    for cat_id, stats in rate_stats_by_category.items():
+        rates = stats["rates"]
+        if rates:
+            rate_stats_final[cat_id] = {
+                "min_net": round(min(rates), 2),
+                "max_net": round(max(rates), 2),
+                "adr_net": round(sum(rates) / len(rates), 2),
+                "rooms": stats["rooms"]
+            }
+
+    occupancy_json = json.dumps({
+        k: decimal_to_float(v) for k, v in occupancy_by_category.items()
+    })
+    revenue_json = json.dumps({
+        k: decimal_to_float(v) for k, v in revenue_by_category.items()
+    })
+    availability_json = json.dumps(availability_by_category)
+    rate_stats_json = json.dumps(rate_stats_final)
+
+    # Upsert into newbook_bookings_stats
+    db.execute(
+        text("""
+            INSERT INTO newbook_bookings_stats (
+                date, rooms_count, maintenance_count, bookable_count,
+                booking_count, guests_count, adults_count, children_count, infants_count,
+                total_occupancy_pct, bookable_occupancy_pct,
+                guest_rate_total, net_booking_rev_total,
+                occupancy_by_category, revenue_by_category, availability_by_category,
+                rate_stats_by_category,
+                aggregated_at
+            ) VALUES (
+                :date, :rooms_count, :maintenance_count, :bookable_count,
+                :booking_count, :guests_count, :adults_count, :children_count, :infants_count,
+                :total_occupancy_pct, :bookable_occupancy_pct,
+                :guest_rate_total, :net_booking_rev_total,
+                :occupancy_by_category, :revenue_by_category, :availability_by_category,
+                :rate_stats_by_category,
+                NOW()
+            )
+            ON CONFLICT (date) DO UPDATE SET
+                rooms_count = :rooms_count,
+                maintenance_count = :maintenance_count,
+                bookable_count = :bookable_count,
+                booking_count = :booking_count,
+                guests_count = :guests_count,
+                adults_count = :adults_count,
+                children_count = :children_count,
+                infants_count = :infants_count,
+                total_occupancy_pct = :total_occupancy_pct,
+                bookable_occupancy_pct = :bookable_occupancy_pct,
+                guest_rate_total = :guest_rate_total,
+                net_booking_rev_total = :net_booking_rev_total,
+                occupancy_by_category = :occupancy_by_category,
+                revenue_by_category = :revenue_by_category,
+                availability_by_category = :availability_by_category,
+                rate_stats_by_category = :rate_stats_by_category,
+                aggregated_at = NOW()
+        """),
+        {
+            "date": target_date,
+            "rooms_count": rooms_count,
+            "maintenance_count": maintenance_count,
+            "bookable_count": bookable_count,
+            "booking_count": booking_count,
+            "guests_count": guests_count,
+            "adults_count": adults_count,
+            "children_count": children_count,
+            "infants_count": infants_count,
+            "total_occupancy_pct": total_occupancy_pct,
+            "bookable_occupancy_pct": bookable_occupancy_pct,
+            "guest_rate_total": float(guest_rate_total),
+            "net_booking_rev_total": float(net_booking_rev_total),
+            "occupancy_by_category": occupancy_json,
+            "revenue_by_category": revenue_json,
+            "availability_by_category": availability_json,
+            "rate_stats_by_category": rate_stats_json
+        }
+    )
+
+
+def get_rate_for_date(raw_json: dict, target_date: date, vat_rate: Decimal) -> tuple:
+    """
+    Extract rate from tariffs_quoted for specific stay_date.
+
+    Returns tuple of (calculated_amount, net_amount).
+    calculated_amount = gross rate guest paid (for AGR)
+    net_amount = amount after VAT deduction
+    """
+    if not raw_json:
+        return Decimal('0'), Decimal('0')
+
+    tariffs = raw_json.get("tariffs_quoted", [])
+    target_str = target_date.strftime("%Y-%m-%d")
+
+    for tariff in tariffs:
+        if tariff.get("stay_date") == target_str:
+            calculated_amount = Decimal(str(tariff.get("calculated_amount", 0) or 0))
+            charge_amount = Decimal(str(tariff.get("charge_amount", 0) or 0))
+
+            # Try to get net from taxes array if available
+            taxes = tariff.get("taxes", [])
+            if taxes and charge_amount > 0:
+                tax_amount = sum(Decimal(str(t.get("tax_amount", 0) or 0)) for t in taxes)
+                net_amount = charge_amount - tax_amount
+            else:
+                # Fallback: calculate net using VAT rate
+                net_amount = charge_amount / (1 + vat_rate)
+
+            return calculated_amount, net_amount
+
+    return Decimal('0'), Decimal('0')
+
+
+async def update_booking_pace(db):
+    """
+    Update booking pace table with current snapshots.
+
+    For each tracked interval, snapshot the current OCCUPANCY count for that stay_date.
+    Occupancy = arrivals + stayovers (guests already checked in from earlier dates).
+
+    This counts bookings where: arrival_date <= stay_date < departure_date
+    Also ensures all dates in the forecast window have rows (prevents gaps when job misses a day).
+    """
+    logger.info("Updating booking pace snapshots (occupancy-based)")
+
+    today = date.today()
+    updates = 0
+
+    # Step 1: Update tracked interval columns
+    for interval in PACE_INTERVALS:
+        stay_date = today + timedelta(days=interval)
+
+        # Count OCCUPANCY for this stay_date (arrivals + stayovers)
+        # A booking occupies a date if: arrival_date <= stay_date < departure_date
+        result = db.execute(
+            text("""
+                SELECT COUNT(*) as count
+                FROM newbook_bookings_data b
+                JOIN newbook_room_categories c ON b.category_id = c.site_id
+                WHERE b.arrival_date <= :stay_date
+                AND b.departure_date > :stay_date
+                AND b.status IN :valid_statuses
+                AND c.is_included = true
+            """),
+            {"stay_date": stay_date, "valid_statuses": VALID_STATUSES}
+        )
+        row = result.fetchone()
+        booking_count = row.count if row else 0
+
+        # Upsert to pace table (column still named arrival_date for backwards compat)
+        column_name = f"d{interval}"
+
+        # Build dynamic SQL for upsert
+        db.execute(
+            text(f"""
+                INSERT INTO newbook_booking_pace (arrival_date, {column_name}, updated_at)
+                VALUES (:stay_date, :count, NOW())
+                ON CONFLICT (arrival_date) DO UPDATE
+                SET {column_name} = :count, updated_at = NOW()
+            """),
+            {"stay_date": stay_date, "count": booking_count}
+        )
+        updates += 1
+
+    # Step 2: Update gap dates (31-36, 38-43, etc.) with their bracketed column
+    # These dates fall between tracked intervals and need their nearest column updated
+    gap_updates = 0
+    for days_out in range(31, 90):  # Cover the gap range where intervals are weekly
+        if days_out in PACE_INTERVALS:
+            continue  # Already handled in step 1
+
+        stay_date = today + timedelta(days=days_out)
+
+        # Find the bracketed column (round up to next interval)
+        bracket_col = None
+        for interval in sorted(PACE_INTERVALS):
+            if interval >= days_out:
+                bracket_col = f"d{interval}"
+                break
+
+        if not bracket_col:
+            continue
+
+        # Count OCCUPANCY (arrivals + stayovers)
+        result = db.execute(
+            text("""
+                SELECT COUNT(*) as count
+                FROM newbook_bookings_data b
+                JOIN newbook_room_categories c ON b.category_id = c.site_id
+                WHERE b.arrival_date <= :stay_date
+                AND b.departure_date > :stay_date
+                AND b.status IN :valid_statuses
+                AND c.is_included = true
+            """),
+            {"stay_date": stay_date, "valid_statuses": VALID_STATUSES}
+        )
+        row = result.fetchone()
+        booking_count = row.count if row else 0
+
+        # Upsert with the bracketed column
+        db.execute(
+            text(f"""
+                INSERT INTO newbook_booking_pace (arrival_date, {bracket_col}, updated_at)
+                VALUES (:stay_date, :count, NOW())
+                ON CONFLICT (arrival_date) DO UPDATE
+                SET {bracket_col} = :count, updated_at = NOW()
+            """),
+            {"stay_date": stay_date, "count": booking_count}
+        )
+        gap_updates += 1
+
+    logger.info(f"Updated {updates} pace snapshots + {gap_updates} gap dates (occupancy-based)")
+
+
+async def fill_occupancy_only_dates(db, vat_rate: Decimal = None):
+    """
+    Create stats rows for dates that have occupancy data but no bookings.
+
+    This ensures dates like closed periods (all rooms in maintenance) get proper
+    stats rows with bookable_count=0, so forecasts can cap correctly.
+    """
+    if vat_rate is None:
+        vat_rate_str = get_config_value(db, 'accommodation_vat_rate')
+        vat_rate = Decimal(vat_rate_str) if vat_rate_str else Decimal('0.20')
+
+    # Find dates with occupancy data but no stats row
+    result = db.execute(
+        text("""
+            SELECT DISTINCT o.date
+            FROM newbook_occupancy_report_data o
+            JOIN newbook_room_categories c ON o.category_id = c.site_id
+            WHERE c.is_included = true
+            AND NOT EXISTS (
+                SELECT 1 FROM newbook_bookings_stats s WHERE s.date = o.date
+            )
+            ORDER BY o.date
+        """)
+    )
+    missing_dates = [row.date for row in result.fetchall()]
+
+    if not missing_dates:
+        logger.info("No occupancy-only dates to fill")
+        return 0
+
+    logger.info(f"Filling {len(missing_dates)} occupancy-only dates (no bookings)")
+
+    for target_date in missing_dates:
+        await aggregate_date(db, target_date, vat_rate)
+
+    logger.info(f"Filled {len(missing_dates)} occupancy-only dates")
+    return len(missing_dates)
+
+
+async def backfill_aggregation(db=None):
+    """
+    Backfill historical data into newbook_bookings_stats and newbook_booking_pace.
+
+    - Stats: Aggregates all dates that have bookings staying
+    - Pace: Reconstructs historical snapshots using booking_placed timestamps
+    """
+    import sys
+    print("[BACKFILL] Starting backfill aggregation...", flush=True)
+    sys.stdout.flush()
+
+    close_db = False
+    if db is None:
+        db = next(iter([SyncSessionLocal()]))
+        close_db = True
+
+    try:
+        # Get VAT rate
+        vat_rate_str = get_config_value(db, 'accommodation_vat_rate')
+        vat_rate = Decimal(vat_rate_str) if vat_rate_str else Decimal('0.20')
+
+        # Step 1: Get all unique stay dates from bookings
+        print("[BACKFILL] Finding all stay dates...", flush=True)
+        result = db.execute(
+            text("""
+                SELECT DISTINCT d::date as stay_date
+                FROM newbook_bookings_data b,
+                     generate_series(b.arrival_date, b.departure_date - interval '1 day', '1 day') d
+                WHERE b.status IN :valid_statuses
+                ORDER BY stay_date
+            """),
+            {"valid_statuses": VALID_STATUSES}
+        )
+        stay_dates = [row.stay_date for row in result.fetchall()]
+        print(f"[BACKFILL] Found {len(stay_dates)} stay dates to aggregate", flush=True)
+
+        # Step 2: Aggregate each stay date into stats
+        for i, target_date in enumerate(stay_dates):
+            if i % 100 == 0:
+                print(f"[BACKFILL] Aggregating stats: {i}/{len(stay_dates)} dates...", flush=True)
+                db.commit()  # Commit periodically
+            await aggregate_date(db, target_date, vat_rate)
+
+        db.commit()
+        print(f"[BACKFILL] Stats aggregation complete: {len(stay_dates)} dates", flush=True)
+
+        # Step 2b: Fill in dates with occupancy data but no bookings (e.g., closed periods)
+        print("[BACKFILL] Filling occupancy-only dates (no bookings)...", flush=True)
+        filled_count = await fill_occupancy_only_dates(db, vat_rate)
+        db.commit()
+        print(f"[BACKFILL] Filled {filled_count} occupancy-only dates", flush=True)
+
+        # Step 3: Get ALL dates from stats for pace backfill
+        # This includes dates with 0 bookings (closed periods, future dates)
+        # Critical: Without pace entries, models may predict 100% occupancy
+        print("[BACKFILL] Finding all stats dates for pace...", flush=True)
+        result = db.execute(
+            text("""
+                SELECT date as stay_date
+                FROM newbook_bookings_stats
+                ORDER BY date
+            """)
+        )
+        stay_dates_for_pace = [row.stay_date for row in result.fetchall()]
+        print(f"[BACKFILL] Found {len(stay_dates_for_pace)} stats dates for pace backfill", flush=True)
+
+        # Step 4: Backfill pace for each stay date (occupancy-based)
+        today = date.today()
+        for i, stay_date in enumerate(stay_dates_for_pace):
+            if i % 100 == 0:
+                print(f"[BACKFILL] Backfilling pace: {i}/{len(stay_dates_for_pace)} dates...", flush=True)
+                db.commit()
+
+            await backfill_pace_for_date(db, stay_date, today)
+
+        db.commit()
+        print(f"[BACKFILL] Pace backfill complete: {len(stay_dates_for_pace)} dates (occupancy-based)", flush=True)
+
+        # Update last aggregation timestamp
+        db.execute(
+            text("""
+                INSERT INTO system_config (config_key, config_value, updated_at)
+                VALUES ('last_bookings_aggregation_at', :now, NOW())
+                ON CONFLICT (config_key) DO UPDATE SET
+                    config_value = :now,
+                    updated_at = NOW()
+            """),
+            {"now": datetime.now().isoformat()}
+        )
+        db.commit()
+
+        print("[BACKFILL] Backfill complete!", flush=True)
+        logger.info("Backfill aggregation completed successfully")
+
+    except Exception as e:
+        print(f"[BACKFILL] FAILED: {e}", flush=True)
+        logger.error(f"Backfill aggregation failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        if close_db:
+            db.close()
+
+
+async def backfill_pace_for_date(db, stay_date: date, today: date):
+    """
+    Backfill pace snapshots for a single stay date using booking_placed timestamps.
+
+    Tracks OCCUPANCY (arrivals + stayovers), not just arrivals.
+    For historical stays: Reconstruct what occupancy would have been at each lead time
+    For future stays: Use current count for today's lead time
+    """
+    # For each interval, calculate what the occupancy count was at that point
+    # Using booking_placed to determine when each booking was created
+    pace_values = {}
+
+    for interval in PACE_INTERVALS:
+        # The snapshot date is when we would have taken this measurement
+        snapshot_date = stay_date - timedelta(days=interval)
+
+        if snapshot_date > today:
+            # This snapshot hasn't happened yet - skip
+            continue
+
+        if snapshot_date < date(2020, 1, 1):
+            # Don't go too far back - skip ancient dates
+            continue
+
+        # Count OCCUPANCY that existed at the snapshot date
+        # A booking contributes to occupancy if:
+        #   - arrival_date <= stay_date < departure_date (booking spans this night)
+        #   - booking_placed <= snapshot_date (booking existed at measurement time)
+        # Only counts categories with is_included = true
+        result = db.execute(
+            text("""
+                SELECT COUNT(*) as count
+                FROM newbook_bookings_data b
+                JOIN newbook_room_categories c ON b.category_id = c.site_id
+                WHERE b.arrival_date <= :stay_date
+                AND b.departure_date > :stay_date
+                AND b.status IN :valid_statuses
+                AND c.is_included = true
+                AND b.booking_placed IS NOT NULL
+                AND b.booking_placed::date <= :snapshot_date
+            """),
+            {
+                "stay_date": stay_date,
+                "valid_statuses": VALID_STATUSES,
+                "snapshot_date": snapshot_date
+            }
+        )
+        row = result.fetchone()
+        pace_values[f"d{interval}"] = row.count if row else 0
+
+    if not pace_values:
+        return
+
+    # Build dynamic upsert for all columns we have values for
+    columns = list(pace_values.keys())
+    set_clauses = ", ".join([f"{col} = :{col}" for col in columns])
+    insert_cols = ", ".join(columns)
+    insert_vals = ", ".join([f":{col}" for col in columns])
+
+    db.execute(
+        text(f"""
+            INSERT INTO newbook_booking_pace (arrival_date, {insert_cols}, updated_at)
+            VALUES (:stay_date, {insert_vals}, NOW())
+            ON CONFLICT (arrival_date) DO UPDATE SET
+                {set_clauses}, updated_at = NOW()
+        """),
+        {"stay_date": stay_date, **pace_values}
+    )
+
+
+async def fill_missing_pace_entries(db=None):
+    """
+    Fill pace entries for all stats dates that don't have pace rows.
+
+    This fixes gaps where dates exist in stats (with 0 or more bookings)
+    but have no pace data, causing models to predict incorrectly.
+    """
+    import sys
+    print("[PACE-FILL] Finding dates missing pace entries...", flush=True)
+
+    close_db = False
+    if db is None:
+        db = next(iter([SyncSessionLocal()]))
+        close_db = True
+
+    try:
+        # Find dates in stats but not in pace
+        result = db.execute(
+            text("""
+                SELECT s.date as stay_date
+                FROM newbook_bookings_stats s
+                LEFT JOIN newbook_booking_pace p ON s.date = p.arrival_date
+                WHERE p.arrival_date IS NULL
+                ORDER BY s.date
+            """)
+        )
+        missing_dates = [row.stay_date for row in result.fetchall()]
+
+        if not missing_dates:
+            print("[PACE-FILL] No missing pace entries found", flush=True)
+            return 0
+
+        print(f"[PACE-FILL] Found {len(missing_dates)} dates missing pace entries", flush=True)
+
+        today = date.today()
+        for i, stay_date in enumerate(missing_dates):
+            if i % 100 == 0:
+                print(f"[PACE-FILL] Processing: {i}/{len(missing_dates)} dates...", flush=True)
+                db.commit()
+
+            await backfill_pace_for_date(db, stay_date, today)
+
+        db.commit()
+        print(f"[PACE-FILL] Filled {len(missing_dates)} missing pace entries", flush=True)
+        return len(missing_dates)
+
+    except Exception as e:
+        print(f"[PACE-FILL] FAILED: {e}", flush=True)
+        db.rollback()
+        raise
+    finally:
+        if close_db:
+            db.close()
+
+
+def get_pace_interval(days_out: int) -> str:
+    """
+    Get the pace column to use for a given lead time.
+    Uses round-up logic (next higher interval for conservative estimates).
+
+    Examples:
+      - 25 days out → d25 (exact daily match)
+      - 35 days out → d37 (rounds up to next weekly)
+      - 200 days out → d210 (rounds up to next monthly)
+    """
+    # Monthly thresholds (7-12 months)
+    if days_out >= 365:
+        return "d365"
+    if days_out >= 330:
+        return "d365"
+    if days_out >= 300:
+        return "d330"
+    if days_out >= 270:
+        return "d300"
+    if days_out >= 240:
+        return "d270"
+    if days_out >= 210:
+        return "d240"
+
+    # Weekly thresholds (5-25 weeks)
+    weekly = [177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
+              107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37]
+    for i, threshold in enumerate(weekly):
+        if days_out >= threshold:
+            return f"d{weekly[i - 1]}" if i > 0 else "d210"
+
+    # Daily (0-30 days) - exact match available
+    if days_out > 30:
+        return "d37"  # Round up to first weekly
+    return f"d{days_out}"

backend/jobs/fetch_current_rates.py

@@ -0,0 +1,359 @@
+"""
+Fetch Current Rates Job
+
+Fetches current rack rates from Newbook API and populates newbook_current_rates table.
+These rates are used by pickup-v2 model for upper bound calculations in confidence shading.
+
+Uses a snapshot model - only inserts new rows when rates change, otherwise updates last_verified_at.
+This allows tracking rate history over time.
+
+Schedule: Daily at 5:20 AM (before pace snapshot runs)
+
+Processing: Day-by-day with progressive DB commits. Each date is fully processed
+(single-night fetch + inline multi-night verification) and saved before moving to the next.
+If the job fails partway, all previously processed dates are preserved.
+"""
+import json
+import logging
+import re
+from datetime import date, timedelta
+from decimal import Decimal
+from typing import Dict, Any, Optional, Set
+
+import asyncio
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+COMMIT_BATCH_SIZE = 10  # Commit to DB every N days
+
+
+def rates_changed(old_rate: Optional[Dict], new_rate: Dict) -> bool:
+    """
+    Compare old and new rates to determine if they've changed.
+
+    Compares gross rate, net rate, and tariff availability status.
+    Returns True if rates have changed, False if they're the same.
+    """
+    if old_rate is None:
+        return True  # No existing rate, need to insert
+
+    # Compare gross and net rates
+    old_gross = float(old_rate.get('rate_gross') or 0)
+    new_gross = float(new_rate.get('gross_rate') or 0)
+    if abs(old_gross - new_gross) > 0.01:
+        return True
+
+    old_net = float(old_rate.get('rate_net') or 0)
+    new_net = float(new_rate.get('net_rate') or 0)
+    if abs(old_net - new_net) > 0.01:
+        return True
+
+    # Compare tariff availability
+    old_tariffs = old_rate.get('tariffs_data', {})
+    if isinstance(old_tariffs, str):
+        try:
+            old_tariffs = json.loads(old_tariffs)
+        except json.JSONDecodeError:
+            old_tariffs = {}
+
+    new_tariffs = new_rate.get('tariffs_data', {})
+
+    old_tariff_list = old_tariffs.get('tariffs', [])
+    new_tariff_list = new_tariffs.get('tariffs', [])
+
+    # Different number of tariffs
+    if len(old_tariff_list) != len(new_tariff_list):
+        return True
+
+    # Compare each tariff's key attributes
+    for old_t, new_t in zip(old_tariff_list, new_tariff_list):
+        # Name changed
+        if old_t.get('name') != new_t.get('name'):
+            return True
+        # Availability status changed
+        if old_t.get('success') != new_t.get('success'):
+            return True
+        # Rate changed significantly
+        old_rate_val = float(old_t.get('rate') or 0)
+        new_rate_val = float(new_t.get('rate') or 0)
+        if abs(old_rate_val - new_rate_val) > 0.01:
+            return True
+        # Min stay changed
+        if old_t.get('min_stay') != new_t.get('min_stay'):
+            return True
+        # Multi-night availability changed
+        if old_t.get('available_for_min_stay') != new_t.get('available_for_min_stay'):
+            return True
+
+    return False
+
+
+def save_rate_snapshot(db, category_id: str, rate_date: date, rate: Dict) -> str:
+    """
+    Save rate to database using snapshot logic.
+
+    If rate has changed from latest version, insert new row.
+    If rate is the same, just update last_verified_at.
+
+    Returns: 'inserted', 'verified', or 'error'
+    """
+    gross_rate = rate.get('gross_rate')
+    net_rate = rate.get('net_rate')
+    tariffs_data = rate.get('tariffs_data', {})
+
+    # Get the latest rate for this category/date
+    existing = db.execute(
+        text("""
+            SELECT id, rate_gross, rate_net, tariffs_data
+            FROM newbook_current_rates
+            WHERE category_id = :category_id AND rate_date = :rate_date
+            ORDER BY valid_from DESC
+            LIMIT 1
+        """),
+        {"category_id": category_id, "rate_date": rate_date}
+    ).fetchone()
+
+    if existing:
+        existing_dict = {
+            'rate_gross': existing.rate_gross,
+            'rate_net': existing.rate_net,
+            'tariffs_data': existing.tariffs_data
+        }
+    else:
+        existing_dict = None
+
+    if rates_changed(existing_dict, rate):
+        # Rates changed - insert new snapshot
+        db.execute(
+            text("""
+                INSERT INTO newbook_current_rates
+                (category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from, last_verified_at)
+                VALUES (:category_id, :rate_date, :rate_gross, :rate_net,
+                        CAST(:tariffs_data AS jsonb), NOW(), NOW())
+            """),
+            {
+                "category_id": category_id,
+                "rate_date": rate_date,
+                "rate_gross": gross_rate,
+                "rate_net": net_rate,
+                "tariffs_data": json.dumps(tariffs_data)
+            }
+        )
+        return 'inserted'
+    else:
+        # Rates unchanged - just verify
+        db.execute(
+            text("""
+                UPDATE newbook_current_rates
+                SET last_verified_at = NOW()
+                WHERE id = :id
+            """),
+            {"id": existing.id}
+        )
+        return 'verified'
+
+
+def needs_multi_night_check(tariff: Dict, days_ahead: int) -> Optional[int]:
+    """
+    Check if a tariff needs multi-night verification.
+
+    Returns the min_stay value if a multi-night check is needed, None otherwise.
+    Skips tariffs with advance booking restrictions that aren't met.
+    """
+    min_stay = tariff.get('min_stay')
+    if not min_stay or min_stay <= 1:
+        return None
+    if tariff.get('success', False):
+        return None  # Already available as single-night, no recheck needed
+
+    # Check for advance booking requirement
+    message = tariff.get('message', '') or ''
+    advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
+    if advance_match:
+        min_advance_days = int(advance_match.group(1))
+        if days_ahead < min_advance_days:
+            return None  # Within advance period - recheck won't help
+
+    return min_stay
+
+
+async def run_fetch_current_rates(horizon_days: int = 720, start_date: date = None):
+    """
+    Fetch current rates for all included categories and store in database.
+
+    Args:
+        horizon_days: Number of days ahead to fetch (default 720 for scheduled, configurable for manual)
+        start_date: Start date for fetch (default today)
+
+    Processing: Day-by-day with progressive commits.
+    For each date:
+      1. Fetch single-night rates (all categories in one API call)
+      2. Check if any tariffs need multi-night verification
+      3. If so, run multi-night check immediately for that date
+      4. Save all rates for that date to DB
+      5. Commit every COMMIT_BATCH_SIZE days
+
+    This means if the job fails at day 400, the first 390+ days are already saved.
+    """
+    logger.info(f"Starting current rates fetch ({horizon_days} days)")
+
+    db = next(iter([SyncSessionLocal()]))
+    today = start_date or date.today()
+
+    try:
+        # Get VAT rate from config
+        vat_result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
+        ).fetchone()
+        vat_rate_str = vat_result.config_value if vat_result and vat_result.config_value else '0.20'
+
+        # Get all included room categories
+        cat_result = db.execute(
+            text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
+        )
+        included_categories = set(row.site_id for row in cat_result.fetchall())
+
+        if not included_categories:
+            logger.warning("No included room categories found")
+            return
+
+        logger.info(f"Fetching rates for {len(included_categories)} categories")
+
+        # Import rates client
+        import base64
+        from services.newbook_rates_client import NewbookRatesClient
+
+        # Get credentials from config (decrypt encrypted values)
+        config_result = db.execute(
+            text("""
+                SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted
+                FROM system_config
+                WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region')
+            """)
+        )
+        config = {}
+        for row in config_result.fetchall():
+            value = row.config_value
+            if row.is_encrypted and value:
+                try:
+                    value = base64.b64decode(value.encode()).decode()
+                except Exception:
+                    pass  # Use raw value if decryption fails
+            config[row.config_key] = value
+
+        if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
+            logger.error("Newbook credentials not configured")
+            return
+
+        # Create client
+        client = NewbookRatesClient(
+            api_key=config['newbook_api_key'],
+            username=config['newbook_username'],
+            password=config['newbook_password'],
+            region=config['newbook_region'],
+            vat_rate=Decimal(vat_rate_str)
+        )
+
+        async with client:
+            inserted_total = 0
+            verified_total = 0
+            multi_night_checks = 0
+            skipped_advance = 0
+            current_date = today
+            day_count = 0
+
+            while current_date <= today + timedelta(days=horizon_days):
+                day_count += 1
+                days_ahead = (current_date - today).days
+
+                try:
+                    # Step 1: Fetch single-night rates for all categories on this date
+                    day_rates = await client.fetch_single_date_all_categories(
+                        current_date, guests_adults=2, guests_children=0
+                    )
+
+                    # Step 2: Check for multi-night verification needs and run inline
+                    # Collect unique min_stay values needed for this date
+                    nights_needed: Set[int] = set()
+                    for cat_id, rates in day_rates.items():
+                        if cat_id not in included_categories:
+                            continue
+                        for rate in rates:
+                            for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
+                                check = needs_multi_night_check(tariff, days_ahead)
+                                if check:
+                                    nights_needed.add(check)
+                                elif tariff.get('min_stay') and tariff['min_stay'] > 1 and not tariff.get('success', False):
+                                    skipped_advance += 1
+
+                    # Step 3: Run multi-night checks for this date if needed
+                    multi_night_results: Dict[int, Dict[str, Dict[str, bool]]] = {}
+                    for nights in sorted(nights_needed):
+                        try:
+                            result = await client.fetch_multi_night_for_date(
+                                current_date, nights
+                            )
+                            multi_night_results[nights] = result
+                            multi_night_checks += 1
+                            await asyncio.sleep(1.0)  # Rate limiting
+                        except Exception as e:
+                            logger.warning(f"Multi-night check failed for {current_date} ({nights}n): {e}")
+
+                    # Step 4: Update tariffs with multi-night results and save to DB
+                    for cat_id, rates in day_rates.items():
+                        if cat_id not in included_categories:
+                            continue
+                        for rate in rates:
+                            tariffs_data = rate.get('tariffs_data', {})
+                            # Apply multi-night results to tariffs
+                            for tariff in tariffs_data.get('tariffs', []):
+                                min_stay = tariff.get('min_stay')
+                                if min_stay and min_stay > 1 and min_stay in multi_night_results:
+                                    cat_availability = multi_night_results[min_stay].get(cat_id, {})
+                                    tariff_name = tariff.get('name', '')
+                                    tariff['available_for_min_stay'] = cat_availability.get(tariff_name, False)
+
+                            # Save to DB
+                            result = save_rate_snapshot(db, cat_id, current_date, rate)
+                            if result == 'inserted':
+                                inserted_total += 1
+                            elif result == 'verified':
+                                verified_total += 1
+
+                    if day_count % 50 == 0 or nights_needed:
+                        logger.info(
+                            f"Day {day_count}/{horizon_days}: {current_date}"
+                            f" | {inserted_total} new, {verified_total} verified"
+                            f"{f' | {len(nights_needed)} multi-night checks' if nights_needed else ''}"
+                        )
+
+                except Exception as e:
+                    logger.warning(f"Failed to fetch rates for {current_date}: {e}")
+
+                # Step 5: Commit periodically
+                if day_count % COMMIT_BATCH_SIZE == 0:
+                    db.commit()
+
+                current_date += timedelta(days=1)
+                await asyncio.sleep(1.0)  # Rate limiting between days
+
+            # Final commit for remaining days
+            db.commit()
+
+            if skipped_advance > 0:
+                logger.info(f"Skipped {skipped_advance} multi-night checks (advance booking restriction)")
+            logger.info(
+                f"Complete: {inserted_total} new snapshots, {verified_total} verified unchanged, "
+                f"{multi_night_checks} multi-night checks"
+            )
+
+        logger.info("Current rates fetch completed successfully")
+
+    except Exception as e:
+        logger.error(f"Current rates fetch failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()

backend/jobs/forecast_daily.py

@@ -0,0 +1,276 @@
+"""
+Daily forecast generation job
+Runs Prophet, XGBoost, and Pickup models
+"""
+import json
+import logging
+import uuid
+from datetime import date, timedelta
+from typing import List, Optional
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+async def run_daily_forecast(
+    horizon_days: int = 14,
+    start_days: int = 0,
+    models: Optional[List[str]] = None,
+    triggered_by: str = "scheduler"
+):
+    """
+    Run daily forecast update for specified horizon.
+
+    Args:
+        horizon_days: How many days ahead to forecast
+        start_days: Start from N days in the future (for medium/long term)
+        models: Which models to run (default: all)
+        triggered_by: Who triggered this run
+    """
+    if models is None:
+        models = ['prophet', 'xgboost', 'pickup', 'catboost']
+
+    run_id = str(uuid.uuid4())
+    forecast_from = date.today() + timedelta(days=start_days)
+    forecast_to = date.today() + timedelta(days=horizon_days)
+
+    logger.info(f"Starting forecast run {run_id}: {forecast_from} to {forecast_to}, models: {models}")
+
+    db = next(iter([SyncSessionLocal()]))
+
+    try:
+        # Log run start
+        db.execute(
+            text("""
+            INSERT INTO forecast_runs (
+                run_id, run_type, started_at, status,
+                forecast_from, forecast_to, models_run, triggered_by
+            ) VALUES (
+                :run_id, 'scheduled', NOW(), 'running',
+                :forecast_from, :forecast_to, :models, :triggered_by
+            )
+            """),
+            {
+                "run_id": run_id,
+                "forecast_from": forecast_from,
+                "forecast_to": forecast_to,
+                "models": json.dumps(models),
+                "triggered_by": triggered_by
+            }
+        )
+        db.commit()
+
+        # Get metrics to forecast
+        result = db.execute(
+            text("""
+            SELECT metric_code, use_prophet, use_xgboost, use_pickup,
+                   COALESCE(use_catboost, TRUE) as use_catboost
+            FROM forecast_metrics
+            WHERE is_active = TRUE
+            """)
+        )
+        metrics = result.fetchall()
+
+        forecasts_generated = 0
+
+        for metric in metrics:
+            metric_code = metric.metric_code
+
+            # Run Prophet if applicable
+            if 'prophet' in models and metric.use_prophet:
+                try:
+                    from services.forecasting.prophet_model import run_prophet_forecast
+                    prophet_forecasts = await run_prophet_forecast(
+                        db, metric_code, forecast_from, forecast_to
+                    )
+                    forecasts_generated += len(prophet_forecasts)
+                except Exception as e:
+                    logger.error(f"Prophet forecast failed for {metric_code}: {e}")
+                    db.rollback()  # Rollback failed transaction
+
+            # Run XGBoost if applicable
+            if 'xgboost' in models and metric.use_xgboost:
+                try:
+                    from services.forecasting.xgboost_model import run_xgboost_forecast
+                    xgboost_forecasts = await run_xgboost_forecast(
+                        db, metric_code, forecast_from, forecast_to
+                    )
+                    forecasts_generated += len(xgboost_forecasts)
+                except Exception as e:
+                    logger.error(f"XGBoost forecast failed for {metric_code}: {e}")
+                    db.rollback()  # Rollback failed transaction
+
+            # Run Pickup if applicable (only for short-term)
+            if 'pickup' in models and metric.use_pickup and start_days < 30:
+                try:
+                    from services.forecasting.pickup_model import run_pickup_forecast
+                    pickup_forecasts = await run_pickup_forecast(
+                        db, metric_code, forecast_from, forecast_to
+                    )
+                    forecasts_generated += len(pickup_forecasts)
+                except Exception as e:
+                    logger.error(f"Pickup forecast failed for {metric_code}: {e}")
+                    db.rollback()  # Rollback failed transaction
+
+            # Run CatBoost if applicable
+            if 'catboost' in models and getattr(metric, 'use_catboost', True):
+                try:
+                    from services.forecasting.catboost_model import run_catboost_forecast
+                    catboost_forecasts = await run_catboost_forecast(
+                        db, metric_code, forecast_from, forecast_to
+                    )
+                    forecasts_generated += len(catboost_forecasts)
+                except Exception as e:
+                    logger.error(f"CatBoost forecast failed for {metric_code}: {e}")
+                    db.rollback()  # Rollback failed transaction
+
+        # Run blended model (accuracy-weighted average of prophet, xgboost, catboost)
+        if 'blended' in models:
+            try:
+                logger.info("Generating blended forecasts with accuracy-based weighting")
+
+                # Get accuracy scores for model weighting (from last 90 days)
+                # Calculate weights per metric
+                metric_weights = {}
+                for metric in metrics:
+                    metric_code = metric.metric_code
+                    try:
+                        accuracy_result = db.execute(
+                            text("""
+                                SELECT
+                                    AVG(ABS(prophet_pct_error)) as prophet_mape,
+                                    AVG(ABS(xgboost_pct_error)) as xgboost_mape,
+                                    AVG(ABS(catboost_pct_error)) as catboost_mape
+                                FROM actual_vs_forecast
+                                WHERE date >= CURRENT_DATE - INTERVAL '90 days'
+                                    AND date < CURRENT_DATE
+                                    AND metric_type = :metric
+                                    AND actual_value IS NOT NULL
+                            """),
+                            {"metric": metric_code}
+                        )
+                        accuracy_row = accuracy_result.fetchone()
+
+                        # Calculate inverse-MAPE weights (lower MAPE = higher weight)
+                        if accuracy_row and accuracy_row.prophet_mape and accuracy_row.xgboost_mape and accuracy_row.catboost_mape:
+                            prophet_mape = float(accuracy_row.prophet_mape) or 10
+                            xgboost_mape = float(accuracy_row.xgboost_mape) or 10
+                            catboost_mape = float(accuracy_row.catboost_mape) or 10
+
+                            inv_prophet = 1 / max(prophet_mape, 0.1)
+                            inv_xgboost = 1 / max(xgboost_mape, 0.1)
+                            inv_catboost = 1 / max(catboost_mape, 0.1)
+                            total_inv = inv_prophet + inv_xgboost + inv_catboost
+
+                            metric_weights[metric_code] = {
+                                'prophet': inv_prophet / total_inv,
+                                'xgboost': inv_xgboost / total_inv,
+                                'catboost': inv_catboost / total_inv
+                            }
+                        else:
+                            # Equal weights if no accuracy data
+                            metric_weights[metric_code] = {'prophet': 1/3, 'xgboost': 1/3, 'catboost': 1/3}
+                    except Exception:
+                        # Default to equal weights on error
+                        metric_weights[metric_code] = {'prophet': 1/3, 'xgboost': 1/3, 'catboost': 1/3}
+
+                # Get all forecasts from the three models for this run
+                result = db.execute(
+                    text("""
+                        SELECT forecast_date, forecast_type, model_type, predicted_value
+                        FROM forecasts
+                        WHERE run_id = :run_id
+                        AND model_type IN ('prophet', 'xgboost', 'catboost')
+                        ORDER BY forecast_date, forecast_type
+                    """),
+                    {"run_id": run_id}
+                )
+                rows = result.fetchall()
+
+                if rows:
+                    # Group by forecast_date and forecast_type
+                    forecasts_by_date_type = {}
+                    for row in rows:
+                        key = (row.forecast_date, row.forecast_type)
+                        if key not in forecasts_by_date_type:
+                            forecasts_by_date_type[key] = {}
+                        forecasts_by_date_type[key][row.model_type] = float(row.predicted_value)
+
+                    # Calculate weighted blended forecast for each date/type combination
+                    blended_count = 0
+                    for (forecast_date, forecast_type), model_forecasts in forecasts_by_date_type.items():
+                        # Only blend if we have at least 2 models
+                        if len(model_forecasts) >= 2:
+                            # Get weights for this metric
+                            weights = metric_weights.get(forecast_type, {'prophet': 1/3, 'xgboost': 1/3, 'catboost': 1/3})
+
+                            # Calculate weighted average
+                            weighted_sum = 0
+                            weight_total = 0
+                            for model, value in model_forecasts.items():
+                                weight = weights.get(model, 0)
+                                weighted_sum += value * weight
+                                weight_total += weight
+
+                            blended_value = weighted_sum / weight_total if weight_total > 0 else sum(model_forecasts.values()) / len(model_forecasts)
+
+                            # Insert blended forecast
+                            db.execute(
+                                text("""
+                                    INSERT INTO forecasts
+                                        (run_id, forecast_date, forecast_type, model_type, predicted_value, generated_at)
+                                    VALUES
+                                        (:run_id, :forecast_date, :forecast_type, 'blended', :predicted_value, NOW())
+                                """),
+                                {
+                                    "run_id": run_id,
+                                    "forecast_date": forecast_date,
+                                    "forecast_type": forecast_type,
+                                    "predicted_value": round(blended_value, 2)
+                                }
+                            )
+                            blended_count += 1
+
+                    db.commit()
+                    forecasts_generated += blended_count
+                    logger.info(f"Generated {blended_count} accuracy-weighted blended forecasts")
+                else:
+                    logger.warning("No individual model forecasts found for blending")
+
+            except Exception as e:
+                logger.error(f"Blended forecast generation failed: {e}")
+
+        # Update run status
+        db.execute(
+            text("""
+            UPDATE forecast_runs
+            SET completed_at = NOW(), status = 'success'
+            WHERE run_id = :run_id
+            """),
+            {"run_id": run_id}
+        )
+        db.commit()
+
+        logger.info(f"Forecast run {run_id} completed: {forecasts_generated} forecasts generated")
+
+    except Exception as e:
+        logger.error(f"Forecast run {run_id} failed: {e}")
+        # Rollback the failed transaction first
+        db.rollback()
+        try:
+            db.execute(
+                text("""
+                UPDATE forecast_runs
+                SET completed_at = NOW(), status = 'failed', error_message = :error
+                WHERE run_id = :run_id
+                """),
+                {"run_id": run_id, "error": str(e)}
+            )
+            db.commit()
+        except Exception as update_error:
+            logger.error(f"Failed to update error status: {update_error}")
+        raise
+    finally:
+        db.close()

backend/jobs/metrics_aggregation.py

@@ -0,0 +1,150 @@
+"""
+Metrics aggregation job for forecast_data database
+Populates daily_metrics from newbook_bookings_stats
+
+This is the data source for forecasting models (Prophet, XGBoost, CatBoost).
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+async def run_metrics_aggregation(
+    from_date: Optional[date] = None,
+    to_date: Optional[date] = None
+):
+    """
+    Populate daily_metrics table from newbook_bookings_stats.
+
+    This provides the historical actuals needed for forecasting models.
+
+    Args:
+        from_date: Start date (defaults to 2 years ago)
+        to_date: End date (defaults to yesterday)
+    """
+    logger.info("Starting metrics aggregation job")
+
+    db = next(iter([SyncSessionLocal()]))
+
+    try:
+        # Default date range: 2 years of history
+        if from_date is None:
+            from_date = date.today() - timedelta(days=730)
+        if to_date is None:
+            to_date = date.today() - timedelta(days=1)
+
+        logger.info(f"Aggregating metrics from {from_date} to {to_date}")
+
+        # Get data from newbook_bookings_stats
+        result = db.execute(
+            text("""
+            SELECT
+                date,
+                booking_count,           -- room nights (occupied rooms)
+                total_occupancy_pct,     -- occupancy percentage
+                guests_count,            -- total guests
+                adults_count,
+                children_count,
+                rooms_count,             -- available rooms
+                bookable_count           -- bookable rooms (rooms - maintenance)
+            FROM newbook_bookings_stats
+            WHERE date BETWEEN :from_date AND :to_date
+            ORDER BY date
+            """),
+            {"from_date": from_date, "to_date": to_date}
+        )
+        stats_rows = result.fetchall()
+
+        if not stats_rows:
+            logger.warning("No data found in newbook_bookings_stats")
+            return
+
+        logger.info(f"Found {len(stats_rows)} days of data to aggregate")
+
+        # Metrics to populate
+        metrics_count = 0
+
+        for row in stats_rows:
+            d = row.date
+
+            # Define metrics from newbook_bookings_stats
+            metrics_to_insert = []
+
+            # Room nights (occupied rooms)
+            if row.booking_count is not None:
+                metrics_to_insert.append(("hotel_room_nights", row.booking_count))
+
+            # Occupancy percentage
+            if row.total_occupancy_pct is not None:
+                metrics_to_insert.append(("hotel_occupancy_pct", float(row.total_occupancy_pct)))
+
+            # Guest count
+            if row.guests_count is not None:
+                metrics_to_insert.append(("hotel_guests", row.guests_count))
+
+            # Insert/update all metrics
+            for metric_code, actual_value in metrics_to_insert:
+                db.execute(
+                    text("""
+                    INSERT INTO daily_metrics (date, metric_code, actual_value, source, updated_at)
+                    VALUES (:date, :metric_code, :actual_value, 'newbook', NOW())
+                    ON CONFLICT (date, metric_code) DO UPDATE SET
+                        actual_value = :actual_value,
+                        updated_at = NOW()
+                    """),
+                    {
+                        "date": d,
+                        "metric_code": metric_code,
+                        "actual_value": actual_value
+                    }
+                )
+                metrics_count += 1
+
+        db.commit()
+        logger.info(f"Aggregated {metrics_count} metric records from {len(stats_rows)} days")
+
+    except Exception as e:
+        logger.error(f"Metrics aggregation failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+async def backfill_daily_metrics():
+    """
+    Backfill all available history from newbook_bookings_stats to daily_metrics.
+    Call this once when setting up forecasting on forecast_data database.
+    """
+    logger.info("Starting full backfill of daily_metrics")
+
+    db = next(iter([SyncSessionLocal()]))
+
+    try:
+        # Find the earliest date in newbook_bookings_stats
+        result = db.execute(text("SELECT MIN(date) as min_date FROM newbook_bookings_stats"))
+        row = result.fetchone()
+
+        if not row or not row.min_date:
+            logger.warning("No data in newbook_bookings_stats to backfill")
+            return
+
+        from_date = row.min_date
+        to_date = date.today() - timedelta(days=1)
+
+        logger.info(f"Backfilling from {from_date} to {to_date}")
+
+        await run_metrics_aggregation(from_date=from_date, to_date=to_date)
+
+        logger.info("Backfill completed successfully")
+
+    except Exception as e:
+        logger.error(f"Backfill failed: {e}")
+        raise
+    finally:
+        db.close()

backend/jobs/pace_snapshot_v2.py

@@ -0,0 +1,439 @@
+"""
+Pace Snapshot V2 - Enhanced pace capture for pickup-v2 model
+
+Captures:
+1. Per-category room counts at each lead time (category_booking_pace)
+2. Total booked accommodation revenue at each lead time (revenue_pace)
+
+This job runs alongside the existing pickup_snapshot job.
+Uses 364-day offset for prior year comparison (52 weeks = day-of-week alignment).
+"""
+import logging
+from datetime import date, timedelta
+from decimal import Decimal
+from typing import Dict, List, Optional
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+# Valid booking statuses for aggregation
+VALID_STATUSES = ('Unconfirmed', 'Confirmed', 'Arrived', 'Departed')
+
+# All tracked pace intervals (same as booking_pace table structure)
+PACE_INTERVALS = [
+    # Monthly (months 7-12)
+    365, 330, 300, 270, 240, 210,
+    # Weekly (weeks 5-25)
+    177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
+    107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
+    # Daily (days 0-30)
+    30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
+    20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
+    10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+]
+
+
+def get_lead_time_column(lead_days: int) -> str:
+    """
+    Map lead days to the appropriate column in pace tables.
+    Uses round-up logic for days between tracked intervals.
+    """
+    if lead_days <= 0:
+        return "d0"
+    elif lead_days <= 30:
+        return f"d{lead_days}"
+    elif lead_days <= 177:
+        # Weekly intervals - find next higher
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        for col in weekly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d177"
+    else:
+        # Monthly intervals
+        monthly_cols = [210, 240, 270, 300, 330, 365]
+        for col in monthly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d365"
+
+
+def get_rate_for_date(raw_json: dict, target_date: date, vat_rate: Decimal) -> Decimal:
+    """
+    Extract net accommodation rate from tariffs_quoted for a specific stay_date.
+    Returns net amount (after VAT deduction).
+    """
+    if not raw_json:
+        return Decimal('0')
+
+    tariffs = raw_json.get("tariffs_quoted", [])
+    target_str = target_date.strftime("%Y-%m-%d")
+
+    for tariff in tariffs:
+        if tariff.get("stay_date") == target_str:
+            charge_amount = Decimal(str(tariff.get("charge_amount", 0) or 0))
+
+            # Try to get net from taxes array if available
+            taxes = tariff.get("taxes", [])
+            if taxes and charge_amount > 0:
+                tax_amount = sum(Decimal(str(t.get("amount", 0) or 0)) for t in taxes)
+                net_amount = charge_amount - tax_amount
+            else:
+                # Fallback: calculate net using VAT rate
+                net_amount = charge_amount / (1 + vat_rate)
+
+            return net_amount
+
+    return Decimal('0')
+
+
+async def run_pace_snapshot_v2():
+    """
+    Capture per-category room counts and total revenue at each lead time.
+
+    Updates:
+    - category_booking_pace: room counts by category for each future date
+    - revenue_pace: total booked accommodation revenue for each future date
+    """
+    logger.info("Starting pace snapshot v2 capture")
+
+    db = next(iter([SyncSessionLocal()]))
+    today = date.today()
+
+    try:
+        # Get accommodation VAT rate from config
+        vat_result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
+        ).fetchone()
+        vat_rate = Decimal(vat_result.config_value) if vat_result and vat_result.config_value else Decimal('0.20')
+
+        # Get all included room categories
+        cat_result = db.execute(
+            text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
+        )
+        included_categories = [row.site_id for row in cat_result.fetchall()]
+
+        if not included_categories:
+            logger.warning("No included room categories found")
+            return
+
+        # Process each tracked interval
+        for interval in PACE_INTERVALS:
+            stay_date = today + timedelta(days=interval)
+            column_name = f"d{interval}"
+
+            # === 1. Capture per-category room counts ===
+            cat_counts = await capture_category_counts(db, stay_date, included_categories)
+
+            for category_id, count in cat_counts.items():
+                db.execute(
+                    text(f"""
+                        INSERT INTO category_booking_pace (arrival_date, category_id, {column_name}, updated_at)
+                        VALUES (:stay_date, :category_id, :count, NOW())
+                        ON CONFLICT (arrival_date, category_id) DO UPDATE
+                        SET {column_name} = :count, updated_at = NOW()
+                    """),
+                    {"stay_date": stay_date, "category_id": category_id, "count": count}
+                )
+
+            # === 2. Capture total booked revenue ===
+            total_revenue = await capture_booked_revenue(db, stay_date, vat_rate, included_categories)
+
+            db.execute(
+                text(f"""
+                    INSERT INTO revenue_pace (stay_date, {column_name}, updated_at)
+                    VALUES (:stay_date, :revenue, NOW())
+                    ON CONFLICT (stay_date) DO UPDATE
+                    SET {column_name} = :revenue, updated_at = NOW()
+                """),
+                {"stay_date": stay_date, "revenue": float(total_revenue)}
+            )
+
+        # Also fill gap dates (31-36, 38-43, etc.) with their bracketed column
+        await fill_gap_dates(db, today, vat_rate, included_categories)
+
+        db.commit()
+        logger.info(f"Pace snapshot v2 completed for {today}")
+
+    except Exception as e:
+        logger.error(f"Pace snapshot v2 failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+async def capture_category_counts(db, stay_date: date, included_categories: List[str]) -> Dict[str, int]:
+    """
+    Count rooms booked per category for a given stay date.
+    Returns dict of {category_id: count}
+    """
+    result = db.execute(
+        text("""
+            SELECT category_id, COUNT(*) as count
+            FROM newbook_bookings_data
+            WHERE arrival_date <= :stay_date
+            AND departure_date > :stay_date
+            AND status IN :valid_statuses
+            AND category_id IN :categories
+            GROUP BY category_id
+        """),
+        {"stay_date": stay_date, "valid_statuses": VALID_STATUSES, "categories": tuple(included_categories)}
+    )
+
+    counts = {cat: 0 for cat in included_categories}  # Initialize all categories with 0
+    for row in result.fetchall():
+        counts[row.category_id] = row.count
+
+    return counts
+
+
+async def capture_booked_revenue(db, stay_date: date, vat_rate: Decimal, included_categories: List[str]) -> Decimal:
+    """
+    Calculate total booked accommodation revenue (net) for a given stay date.
+    Sums up tariffs from all active bookings that span this date.
+    """
+    result = db.execute(
+        text("""
+            SELECT raw_json
+            FROM newbook_bookings_data
+            WHERE arrival_date <= :stay_date
+            AND departure_date > :stay_date
+            AND status IN :valid_statuses
+            AND category_id IN :categories
+        """),
+        {"stay_date": stay_date, "valid_statuses": VALID_STATUSES, "categories": tuple(included_categories)}
+    )
+
+    total_revenue = Decimal('0')
+    for row in result.fetchall():
+        if row.raw_json:
+            revenue = get_rate_for_date(row.raw_json, stay_date, vat_rate)
+            total_revenue += revenue
+
+    return total_revenue
+
+
+async def fill_gap_dates(db, today: date, vat_rate: Decimal, included_categories: List[str]):
+    """
+    Fill gap dates (between tracked intervals) with their bracketed column value.
+    These dates fall between weekly intervals and need the next higher column updated.
+    """
+    gap_updates = 0
+
+    for days_out in range(31, 90):  # Cover the gap range where intervals are weekly
+        if days_out in PACE_INTERVALS:
+            continue  # Already handled in main loop
+
+        stay_date = today + timedelta(days=days_out)
+        bracket_col = get_lead_time_column(days_out)
+
+        # Capture category counts
+        cat_counts = await capture_category_counts(db, stay_date, included_categories)
+        for category_id, count in cat_counts.items():
+            db.execute(
+                text(f"""
+                    INSERT INTO category_booking_pace (arrival_date, category_id, {bracket_col}, updated_at)
+                    VALUES (:stay_date, :category_id, :count, NOW())
+                    ON CONFLICT (arrival_date, category_id) DO UPDATE
+                    SET {bracket_col} = :count, updated_at = NOW()
+                """),
+                {"stay_date": stay_date, "category_id": category_id, "count": count}
+            )
+
+        # Capture revenue
+        total_revenue = await capture_booked_revenue(db, stay_date, vat_rate, included_categories)
+        db.execute(
+            text(f"""
+                INSERT INTO revenue_pace (stay_date, {bracket_col}, updated_at)
+                VALUES (:stay_date, :revenue, NOW())
+                ON CONFLICT (stay_date) DO UPDATE
+                SET {bracket_col} = :revenue, updated_at = NOW()
+            """),
+            {"stay_date": stay_date, "revenue": float(total_revenue)}
+        )
+
+        gap_updates += 1
+
+    logger.info(f"Filled {gap_updates} gap dates for category pace and revenue pace")
+
+
+async def backfill_pace_v2(db=None):
+    """
+    Backfill historical pace v2 data using booking_placed timestamps.
+
+    Reconstructs what category counts and revenue would have been at each lead time
+    for historical dates.
+    """
+    import sys
+    print("[PACE-V2-BACKFILL] Starting backfill...", flush=True)
+
+    close_db = False
+    if db is None:
+        db = next(iter([SyncSessionLocal()]))
+        close_db = True
+
+    try:
+        # Get VAT rate
+        vat_result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
+        ).fetchone()
+        vat_rate = Decimal(vat_result.config_value) if vat_result and vat_result.config_value else Decimal('0.20')
+
+        # Get included categories
+        cat_result = db.execute(
+            text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
+        )
+        included_categories = [row.site_id for row in cat_result.fetchall()]
+
+        if not included_categories:
+            print("[PACE-V2-BACKFILL] No included categories found", flush=True)
+            return
+
+        # Get all unique stay dates from bookings_stats
+        result = db.execute(
+            text("""
+                SELECT date as stay_date
+                FROM newbook_bookings_stats
+                WHERE date >= CURRENT_DATE - INTERVAL '2 years'
+                ORDER BY date
+            """)
+        )
+        stay_dates = [row.stay_date for row in result.fetchall()]
+        print(f"[PACE-V2-BACKFILL] Found {len(stay_dates)} dates to process", flush=True)
+
+        today = date.today()
+
+        for i, stay_date in enumerate(stay_dates):
+            if i % 100 == 0:
+                print(f"[PACE-V2-BACKFILL] Processing: {i}/{len(stay_dates)} dates...", flush=True)
+                db.commit()
+
+            await backfill_pace_v2_for_date(db, stay_date, today, vat_rate, included_categories)
+
+        db.commit()
+        print(f"[PACE-V2-BACKFILL] Complete: {len(stay_dates)} dates processed", flush=True)
+
+    except Exception as e:
+        print(f"[PACE-V2-BACKFILL] FAILED: {e}", flush=True)
+        db.rollback()
+        raise
+    finally:
+        if close_db:
+            db.close()
+
+
+async def backfill_pace_v2_for_date(
+    db,
+    stay_date: date,
+    today: date,
+    vat_rate: Decimal,
+    included_categories: List[str]
+):
+    """
+    Backfill pace v2 data for a single date using booking_placed timestamps.
+    """
+    pace_category_values: Dict[str, Dict[str, int]] = {cat: {} for cat in included_categories}
+    pace_revenue_values: Dict[str, Decimal] = {}
+
+    for interval in PACE_INTERVALS:
+        snapshot_date = stay_date - timedelta(days=interval)
+
+        if snapshot_date > today:
+            continue  # This snapshot hasn't happened yet
+        if snapshot_date < date(2020, 1, 1):
+            continue  # Don't go too far back
+
+        column_name = f"d{interval}"
+
+        # Count per-category bookings that existed at snapshot_date
+        result = db.execute(
+            text("""
+                SELECT category_id, COUNT(*) as count
+                FROM newbook_bookings_data
+                WHERE arrival_date <= :stay_date
+                AND departure_date > :stay_date
+                AND status IN :valid_statuses
+                AND category_id IN :categories
+                AND booking_placed IS NOT NULL
+                AND booking_placed::date <= :snapshot_date
+                GROUP BY category_id
+            """),
+            {
+                "stay_date": stay_date,
+                "valid_statuses": VALID_STATUSES,
+                "categories": tuple(included_categories),
+                "snapshot_date": snapshot_date
+            }
+        )
+
+        for row in result.fetchall():
+            pace_category_values[row.category_id][column_name] = row.count
+
+        # Calculate revenue that was booked at snapshot_date
+        result = db.execute(
+            text("""
+                SELECT raw_json
+                FROM newbook_bookings_data
+                WHERE arrival_date <= :stay_date
+                AND departure_date > :stay_date
+                AND status IN :valid_statuses
+                AND category_id IN :categories
+                AND booking_placed IS NOT NULL
+                AND booking_placed::date <= :snapshot_date
+            """),
+            {
+                "stay_date": stay_date,
+                "valid_statuses": VALID_STATUSES,
+                "categories": tuple(included_categories),
+                "snapshot_date": snapshot_date
+            }
+        )
+
+        total_revenue = Decimal('0')
+        for row in result.fetchall():
+            if row.raw_json:
+                revenue = get_rate_for_date(row.raw_json, stay_date, vat_rate)
+                total_revenue += revenue
+
+        pace_revenue_values[column_name] = total_revenue
+
+    # Upsert category pace values
+    for category_id, columns in pace_category_values.items():
+        if not columns:
+            continue
+
+        col_names = list(columns.keys())
+        set_clauses = ", ".join([f"{col} = :{col}" for col in col_names])
+        insert_cols = ", ".join(col_names)
+        insert_vals = ", ".join([f":{col}" for col in col_names])
+
+        db.execute(
+            text(f"""
+                INSERT INTO category_booking_pace (arrival_date, category_id, {insert_cols}, updated_at)
+                VALUES (:stay_date, :category_id, {insert_vals}, NOW())
+                ON CONFLICT (arrival_date, category_id) DO UPDATE SET
+                    {set_clauses}, updated_at = NOW()
+            """),
+            {"stay_date": stay_date, "category_id": category_id, **columns}
+        )
+
+    # Upsert revenue pace values
+    if pace_revenue_values:
+        col_names = list(pace_revenue_values.keys())
+        float_values = {k: float(v) for k, v in pace_revenue_values.items()}
+        set_clauses = ", ".join([f"{col} = :{col}" for col in col_names])
+        insert_cols = ", ".join(col_names)
+        insert_vals = ", ".join([f":{col}" for col in col_names])
+
+        db.execute(
+            text(f"""
+                INSERT INTO revenue_pace (stay_date, {insert_cols}, updated_at)
+                VALUES (:stay_date, {insert_vals}, NOW())
+                ON CONFLICT (stay_date) DO UPDATE SET
+                    {set_clauses}, updated_at = NOW()
+            """),
+            {"stay_date": stay_date, **float_values}
+        )

backend/jobs/pickup_snapshot.py

@@ -0,0 +1,208 @@
+"""
+Pickup snapshot job - captures daily on-the-books values
+Used by the pickup model for pace comparison
+
+Prior year comparison uses 364 days (52 weeks) for day-of-week alignment:
+- Monday compares to Monday
+- Saturday compares to Saturday
+"""
+import logging
+from datetime import date, timedelta
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+from utils.time_alignment import get_prior_year_daily, SQL_PRIOR_YEAR_OFFSET
+
+logger = logging.getLogger(__name__)
+
+
+async def run_pickup_snapshot():
+    """
+    Capture daily on-the-books snapshot for future dates.
+    Stores OTB values at various lead times for pickup model.
+    """
+    logger.info("Starting pickup snapshot capture")
+
+    db = next(iter([SyncSessionLocal()]))
+    snapshot_date = date.today()
+
+    # Overflow room category (category_id=5) is used for chargeable no-shows/cancellations
+    # and should be excluded from room night counts
+    overflow_category_id = '5'
+
+    try:
+        # Capture OTB for next 365 days (extended from 60 for longer-term forecasting)
+        for days_out in range(1, 366):
+            stay_date = snapshot_date + timedelta(days=days_out)
+
+            # Get total available rooms for this date from occupancy report
+            rooms_result = db.execute(
+                text("""
+                SELECT COALESCE(SUM(available), 25) as total_rooms
+                FROM newbook_occupancy_report
+                WHERE date = :stay_date
+                """),
+                {"stay_date": stay_date}
+            ).fetchone()
+            total_rooms = rooms_result.total_rooms if rooms_result and rooms_result.total_rooms else 25
+
+            # Get hotel occupancy OTB (count rooms on the books for this stay date)
+            # EXCLUDES overflow category (chargeable no-shows)
+            hotel_result = db.execute(
+                text("""
+                SELECT COUNT(DISTINCT newbook_id) as bookings,
+                       SUM(CASE WHEN LOWER(status) IN ('confirmed', 'provisional', 'unconfirmed', 'arrived') THEN 1 ELSE 0 END) as active_bookings
+                FROM newbook_bookings
+                WHERE arrival_date <= :stay_date AND departure_date > :stay_date
+                    AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')
+                    AND (category_id IS NULL OR category_id != :overflow_cat)
+                """),
+                {"stay_date": stay_date, "overflow_cat": overflow_category_id}
+            )
+            hotel_row = hotel_result.fetchone()
+            hotel_otb = hotel_row.active_bookings or 0
+
+            # Get dinner covers OTB
+            dinner_result = db.execute(
+                text("""
+                SELECT COUNT(*) as bookings, COALESCE(SUM(covers), 0) as total_covers
+                FROM resos_bookings
+                WHERE booking_date = :stay_date
+                    AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show')
+                    AND booking_time >= '15:00'
+                """),
+                {"stay_date": stay_date}
+            )
+            dinner_row = dinner_result.fetchone()
+            dinner_otb = dinner_row.total_covers or 0
+
+            # Get lunch covers OTB
+            lunch_result = db.execute(
+                text("""
+                SELECT COUNT(*) as bookings, COALESCE(SUM(covers), 0) as total_covers
+                FROM resos_bookings
+                WHERE booking_date = :stay_date
+                    AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show')
+                    AND booking_time < '15:00'
+                """),
+                {"stay_date": stay_date}
+            )
+            lunch_row = lunch_result.fetchone()
+            lunch_otb = lunch_row.total_covers or 0
+
+            # Get prior year comparison data (same day of week, exactly 52 weeks ago)
+            # Uses SQL_PRIOR_YEAR_OFFSET (364 days = 52 weeks) for Mon→Mon, Sat→Sat alignment
+            prior_year_stay_date = get_prior_year_daily(stay_date)
+            prior_year_snapshot_date = get_prior_year_daily(snapshot_date)  # Same lead time last year
+
+            # Calculate prior year hotel OTB at same lead time using booking_placed
+            # EXCLUDES overflow category
+            prior_hotel_otb_result = db.execute(
+                text("""
+                SELECT COUNT(DISTINCT newbook_id) as otb_count
+                FROM newbook_bookings
+                WHERE arrival_date <= :prior_stay_date
+                    AND departure_date > :prior_stay_date
+                    AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')
+                    AND (raw_json->>'booking_placed')::timestamp <= :prior_snapshot_date
+                    AND (category_id IS NULL OR category_id != :overflow_cat)
+                """),
+                {"prior_stay_date": prior_year_stay_date, "prior_snapshot_date": prior_year_snapshot_date, "overflow_cat": overflow_category_id}
+            ).fetchone()
+            prior_hotel_otb = prior_hotel_otb_result.otb_count if prior_hotel_otb_result else None
+
+            # Store snapshots for each metric
+            # For hotel_occupancy_pct: store room count, convert to % for otb_value
+            # For hotel_room_nights: store raw room count (no conversion)
+            # For restaurant: store cover counts directly
+            for metric_type, otb_raw, otb_bookings_count in [
+                ('hotel_occupancy_pct', hotel_otb, hotel_row.bookings),
+                ('hotel_room_nights', hotel_otb, hotel_row.bookings),  # Same count, stored as-is
+                ('resos_dinner_covers', dinner_otb, dinner_row.bookings),
+                ('resos_lunch_covers', lunch_otb, lunch_row.bookings)
+            ]:
+                # Convert to percentage for occupancy metric, keep raw counts for others
+                if metric_type == 'hotel_occupancy_pct':
+                    otb_value = (otb_raw / total_rooms) * 100 if total_rooms > 0 else 0
+                    # Use 'is not None' check - 0 is valid data (no bookings at that lead time)
+                    prior_otb_value = (prior_hotel_otb / total_rooms) * 100 if prior_hotel_otb is not None and total_rooms > 0 else None
+                elif metric_type == 'hotel_room_nights':
+                    otb_value = otb_raw  # Raw room count
+                    prior_otb_value = prior_hotel_otb  # Raw room count from prior year
+                else:
+                    otb_value = otb_raw
+                    prior_otb_value = None  # Will try historical snapshots below
+
+                # Get prior year ACTUAL from daily_metrics (the final outcome)
+                prior_final_result = db.execute(
+                    text("""
+                    SELECT actual_value
+                    FROM daily_metrics
+                    WHERE date = :prior_date AND metric_code = :metric
+                    """),
+                    {"prior_date": prior_year_stay_date, "metric": metric_type}
+                ).fetchone()
+                prior_final = float(prior_final_result.actual_value) if prior_final_result and prior_final_result.actual_value else None
+
+                # Use reconstructed prior year OTB for hotel metrics, or try historical snapshots
+                if metric_type in ('hotel_occupancy_pct', 'hotel_room_nights') and prior_otb_value is not None:
+                    prior_otb = prior_otb_value
+                else:
+                    # For restaurant metrics, fall back to historical snapshots
+                    prior_otb_result = db.execute(
+                        text("""
+                        SELECT otb_value
+                        FROM pickup_snapshots
+                        WHERE stay_date = :prior_stay_date
+                            AND metric_type = :metric
+                            AND days_out = :days_out
+                        ORDER BY snapshot_date DESC LIMIT 1
+                        """),
+                        {"prior_stay_date": prior_year_stay_date, "metric": metric_type, "days_out": days_out}
+                    ).fetchone()
+                    # Use 'is not None' - 0 is valid OTB data
+                    prior_otb = float(prior_otb_result.otb_value) if prior_otb_result and prior_otb_result.otb_value is not None else None
+
+                # Calculate pace vs prior year if we have comparison data
+                pace_pct = None
+                if prior_otb and prior_otb > 0:
+                    pace_pct = ((otb_value - prior_otb) / prior_otb) * 100
+
+                db.execute(
+                    text("""
+                    INSERT INTO pickup_snapshots (
+                        snapshot_date, stay_date, days_out, metric_type,
+                        otb_value, otb_bookings, prior_year_otb, prior_year_final,
+                        pace_vs_prior_pct, created_at
+                    ) VALUES (
+                        :snapshot_date, :stay_date, :days_out, :metric_type,
+                        :otb_value, :otb_bookings, :prior_year_otb, :prior_year_final,
+                        :pace_pct, NOW()
+                    )
+                    ON CONFLICT (snapshot_date, stay_date, metric_type) DO UPDATE SET
+                        otb_value = :otb_value,
+                        prior_year_otb = :prior_year_otb,
+                        pace_vs_prior_pct = :pace_pct
+                    """),
+                    {
+                        "snapshot_date": snapshot_date,
+                        "stay_date": stay_date,
+                        "days_out": days_out,
+                        "metric_type": metric_type,
+                        "otb_value": round(otb_value, 2),  # % for hotel, covers for restaurant
+                        "otb_bookings": otb_raw,  # Raw count (rooms or covers)
+                        "prior_year_otb": round(prior_otb, 2) if prior_otb is not None else None,
+                        "prior_year_final": prior_final,
+                        "pace_pct": round(pace_pct, 2) if pace_pct else None
+                    }
+                )
+
+        db.commit()
+        logger.info(f"Pickup snapshot completed for {snapshot_date}")
+
+    except Exception as e:
+        logger.error(f"Pickup snapshot failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()

backend/jobs/resos_aggregation.py

@@ -0,0 +1,476 @@
+"""
+Resos Bookings Aggregation Job
+Aggregates resos_bookings_data into:
+- resos_bookings_stats: daily aggregated stats with period/source breakdowns
+- resos_booking_pace: lead-time snapshots for pickup forecasting (3 types)
+"""
+import json
+import logging
+from datetime import date, datetime, timedelta
+from typing import Set, Dict, Any, Optional, Tuple, List
+from collections import defaultdict
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+# Valid booking statuses for aggregation
+# Note: Resos uses 'approved' for confirmed future reservations, 'left' for completed meals
+VALID_STATUSES = ('approved', 'arrived', 'seated', 'left')
+
+# All tracked pace intervals (same as Newbook)
+PACE_INTERVALS = [
+    # Monthly (months 7-12)
+    365, 330, 300, 270, 240, 210,
+    # Weekly (weeks 5-25)
+    177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
+    107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
+    # Daily (days 0-30)
+    30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
+    20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
+    10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+]
+
+
+def parse_group_exclude_field(group_exclude_field: Optional[str], primary_booking_number: Optional[str]) -> Tuple[List[str], List[str]]:
+    """
+    Parse group_exclude_field to extract linked bookings and exclude markers.
+    Returns: (all_booking_numbers, exclude_numbers)
+    """
+    all_booking_numbers = []
+    exclude_numbers = []
+
+    # Always include primary booking number
+    if primary_booking_number:
+        all_booking_numbers.append(primary_booking_number)
+
+    if not group_exclude_field:
+        return all_booking_numbers, exclude_numbers
+
+    # Parse comma-separated entries
+    parts = group_exclude_field.split(',')
+    for part in parts:
+        part = part.strip()
+
+        if part.upper().startswith('NOT-#'):
+            # Exclude marker: NOT-#56748 → NB56748
+            booking_num = part[5:]  # Remove "NOT-#"
+            exclude_numbers.append(f"NB{booking_num}")
+
+        elif part.startswith('#'):
+            # Additional booking: #12346 → NB12346
+            booking_num = part[1:]  # Remove "#"
+            all_booking_numbers.append(f"NB{booking_num}")
+
+    return all_booking_numbers, exclude_numbers
+
+
+async def aggregate_resos_bookings(triggered_by: str = "manual"):
+    """
+    Aggregate Resos bookings into resos_bookings_stats.
+
+    Flow:
+    1. Find bookings changed since last_resos_aggregation_at
+    2. Calculate affected dates
+    3. Reaggregate each affected date
+    4. Update booking pace table (3 types)
+    5. Update last_resos_aggregation_at
+    """
+    logger.info(f"Starting Resos bookings aggregation (triggered_by={triggered_by})")
+
+    db = SyncSessionLocal()
+
+    try:
+        # Get last aggregation timestamp
+        result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = 'last_resos_aggregation_at'")
+        )
+        row = result.fetchone()
+        if row and row.config_value:
+            try:
+                last_ts = datetime.fromisoformat(row.config_value)
+            except ValueError:
+                last_ts = datetime.min
+        else:
+            last_ts = datetime.min
+
+        logger.info(f"Last aggregation: {last_ts}")
+
+        # Find bookings changed since last aggregation
+        result = db.execute(
+            text("""
+                SELECT resos_id, booking_date
+                FROM resos_bookings_data
+                WHERE fetched_at > :last_ts
+            """),
+            {"last_ts": last_ts}
+        )
+        changed_bookings = result.fetchall()
+
+        if not changed_bookings:
+            logger.info("No changed bookings to aggregate")
+            # Still update pace table
+            await update_resos_booking_pace(db)
+            db.commit()
+
+            # Update timestamp
+            db.execute(
+                text("""
+                    INSERT INTO system_config (config_key, config_value, updated_at)
+                    VALUES ('last_resos_aggregation_at', :now, NOW())
+                    ON CONFLICT (config_key) DO UPDATE SET
+                        config_value = :now,
+                        updated_at = NOW()
+                """),
+                {"now": datetime.now().isoformat()}
+            )
+            db.commit()
+            return
+
+        logger.info(f"Found {len(changed_bookings)} changed bookings")
+
+        # Calculate affected dates
+        affected_dates: Set[date] = set()
+        for booking in changed_bookings:
+            if booking.booking_date:
+                affected_dates.add(booking.booking_date)
+
+        logger.info(f"Reaggregating {len(affected_dates)} affected dates")
+
+        # Aggregate each affected date
+        for target_date in sorted(affected_dates):
+            await aggregate_date(db, target_date)
+
+        # Update booking pace table (3 types)
+        await update_resos_booking_pace(db)
+
+        # Update last aggregation timestamp
+        db.execute(
+            text("""
+                INSERT INTO system_config (config_key, config_value, updated_at)
+                VALUES ('last_resos_aggregation_at', :now, NOW())
+                ON CONFLICT (config_key) DO UPDATE SET
+                    config_value = :now,
+                    updated_at = NOW()
+            """),
+            {"now": datetime.now().isoformat()}
+        )
+
+        db.commit()
+        logger.info(f"Resos bookings aggregation completed: {len(affected_dates)} dates processed")
+
+    except Exception as e:
+        logger.error(f"Resos bookings aggregation failed: {e}", exc_info=True)
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+async def aggregate_date(db, target_date: date):
+    """
+    Aggregate all Resos bookings for a specific date into resos_bookings_stats.
+    """
+    # Get all valid bookings for this date
+    result = db.execute(
+        text("""
+            SELECT
+                resos_id,
+                period_type,
+                covers,
+                source,
+                opening_hour_id,
+                is_hotel_guest,
+                is_dbb,
+                is_package,
+                total_guests,
+                hotel_booking_number,
+                group_exclude_field
+            FROM resos_bookings_data
+            WHERE booking_date = :target_date
+            AND status IN :valid_statuses
+        """),
+        {"target_date": target_date, "valid_statuses": VALID_STATUSES}
+    )
+    bookings = result.fetchall()
+
+    # Initialize counters
+    breakfast_covers = 0
+    lunch_covers = 0
+    afternoon_covers = 0
+    dinner_covers = 0
+    other_covers = 0
+
+    breakfast_bookings = 0
+    lunch_bookings = 0
+    afternoon_bookings = 0
+    dinner_bookings = 0
+    other_bookings = 0
+
+    hotel_guest_covers = 0
+    non_hotel_guest_covers = 0
+    dbb_covers = 0
+    package_covers = 0
+
+    covers_by_source: Dict[str, int] = defaultdict(int)
+    covers_by_period: Dict[str, Dict[str, Any]] = {}
+
+    total_party_sizes = []
+    party_sizes_by_period: Dict[str, list] = defaultdict(list)
+
+    # Build hotel_booking_numbers mapping
+    hotel_booking_numbers: Dict[str, str] = {}  # hotel_booking_number -> resos_id
+    bookings_with_hotel_link = 0
+
+    for booking in bookings:
+        period = booking.period_type or 'other'
+        covers = booking.covers or 0
+        source = booking.source or 'unknown'
+        opening_hour_id = booking.opening_hour_id
+        resos_id = booking.resos_id
+
+        # Count by period
+        if period == 'breakfast':
+            breakfast_covers += covers
+            breakfast_bookings += 1
+        elif period == 'lunch':
+            lunch_covers += covers
+            lunch_bookings += 1
+        elif period == 'afternoon':
+            afternoon_covers += covers
+            afternoon_bookings += 1
+        elif period == 'dinner':
+            dinner_covers += covers
+            dinner_bookings += 1
+        else:
+            other_covers += covers
+            other_bookings += 1
+
+        # Count by source
+        covers_by_source[source] += covers
+
+        # Count by period (detailed)
+        if opening_hour_id:
+            if opening_hour_id not in covers_by_period:
+                covers_by_period[opening_hour_id] = {
+                    "period_type": period,
+                    "covers": 0,
+                    "bookings": 0
+                }
+            covers_by_period[opening_hour_id]["covers"] += covers
+            covers_by_period[opening_hour_id]["bookings"] += 1
+
+        # Count business segments
+        if booking.is_hotel_guest:
+            hotel_guest_covers += covers
+        else:
+            non_hotel_guest_covers += covers
+
+        if booking.is_dbb:
+            dbb_covers += covers
+        if booking.is_package:
+            package_covers += covers
+
+        # Track party sizes
+        if covers > 0:
+            total_party_sizes.append(covers)
+            party_sizes_by_period[period].append(covers)
+
+        # Build hotel booking numbers mapping
+        all_booking_numbers, _ = parse_group_exclude_field(
+            booking.group_exclude_field,
+            booking.hotel_booking_number
+        )
+
+        if all_booking_numbers:
+            bookings_with_hotel_link += 1
+            for hotel_number in all_booking_numbers:
+                hotel_booking_numbers[hotel_number] = resos_id
+
+    # Calculate averages
+    avg_party_size = sum(total_party_sizes) / len(total_party_sizes) if total_party_sizes else None
+
+    avg_by_period = {}
+    for period, sizes in party_sizes_by_period.items():
+        avg_by_period[period] = sum(sizes) / len(sizes) if sizes else None
+
+    total_covers = breakfast_covers + lunch_covers + afternoon_covers + dinner_covers + other_covers
+    total_bookings = breakfast_bookings + lunch_bookings + afternoon_bookings + dinner_bookings + other_bookings
+    distinct_hotel_bookings = len(hotel_booking_numbers)
+
+    # Upsert stats
+    db.execute(
+        text("""
+            INSERT INTO resos_bookings_stats (
+                date,
+                breakfast_covers, lunch_covers, afternoon_covers, dinner_covers, other_covers, total_covers,
+                breakfast_bookings, lunch_bookings, afternoon_bookings, dinner_bookings, other_bookings, total_bookings,
+                covers_by_source, covers_by_period,
+                hotel_guest_covers, non_hotel_guest_covers, dbb_covers, package_covers,
+                hotel_booking_numbers, distinct_hotel_bookings, bookings_with_hotel_link,
+                avg_party_size, avg_party_size_by_period,
+                aggregated_at
+            ) VALUES (
+                :date,
+                :breakfast_covers, :lunch_covers, :afternoon_covers, :dinner_covers, :other_covers, :total_covers,
+                :breakfast_bookings, :lunch_bookings, :afternoon_bookings, :dinner_bookings, :other_bookings, :total_bookings,
+                :covers_by_source, :covers_by_period,
+                :hotel_guest_covers, :non_hotel_guest_covers, :dbb_covers, :package_covers,
+                :hotel_booking_numbers, :distinct_hotel_bookings, :bookings_with_hotel_link,
+                :avg_party_size, :avg_party_size_by_period,
+                NOW()
+            )
+            ON CONFLICT (date) DO UPDATE SET
+                breakfast_covers = :breakfast_covers,
+                lunch_covers = :lunch_covers,
+                afternoon_covers = :afternoon_covers,
+                dinner_covers = :dinner_covers,
+                other_covers = :other_covers,
+                total_covers = :total_covers,
+                breakfast_bookings = :breakfast_bookings,
+                lunch_bookings = :lunch_bookings,
+                afternoon_bookings = :afternoon_bookings,
+                dinner_bookings = :dinner_bookings,
+                other_bookings = :other_bookings,
+                total_bookings = :total_bookings,
+                covers_by_source = :covers_by_source,
+                covers_by_period = :covers_by_period,
+                hotel_guest_covers = :hotel_guest_covers,
+                non_hotel_guest_covers = :non_hotel_guest_covers,
+                dbb_covers = :dbb_covers,
+                package_covers = :package_covers,
+                hotel_booking_numbers = :hotel_booking_numbers,
+                distinct_hotel_bookings = :distinct_hotel_bookings,
+                bookings_with_hotel_link = :bookings_with_hotel_link,
+                avg_party_size = :avg_party_size,
+                avg_party_size_by_period = :avg_party_size_by_period,
+                aggregated_at = NOW()
+        """),
+        {
+            "date": target_date,
+            "breakfast_covers": breakfast_covers,
+            "lunch_covers": lunch_covers,
+            "afternoon_covers": afternoon_covers,
+            "dinner_covers": dinner_covers,
+            "other_covers": other_covers,
+            "total_covers": total_covers,
+            "breakfast_bookings": breakfast_bookings,
+            "lunch_bookings": lunch_bookings,
+            "afternoon_bookings": afternoon_bookings,
+            "dinner_bookings": dinner_bookings,
+            "other_bookings": other_bookings,
+            "total_bookings": total_bookings,
+            "covers_by_source": json.dumps(dict(covers_by_source)),
+            "covers_by_period": json.dumps(covers_by_period),
+            "hotel_guest_covers": hotel_guest_covers,
+            "non_hotel_guest_covers": non_hotel_guest_covers,
+            "dbb_covers": dbb_covers,
+            "package_covers": package_covers,
+            "hotel_booking_numbers": json.dumps(hotel_booking_numbers),
+            "distinct_hotel_bookings": distinct_hotel_bookings,
+            "bookings_with_hotel_link": bookings_with_hotel_link,
+            "avg_party_size": avg_party_size,
+            "avg_party_size_by_period": json.dumps(avg_by_period)
+        }
+    )
+    db.commit()
+
+
+async def update_resos_booking_pace(db):
+    """
+    Update resos_booking_pace table with lead-time snapshots.
+    Creates 3 rows per date: total, resident, non_resident
+    """
+    logger.info("Updating Resos booking pace table...")
+
+    today = date.today()
+
+    # Process dates from -30 to +365 (historical + forecast window)
+    from_date = today - timedelta(days=30)
+    to_date = today + timedelta(days=365)
+
+    current = from_date
+    while current <= to_date:
+        # Calculate pace for each type
+        for pace_type in ['total', 'resident', 'non_resident']:
+            pace_values = {}
+
+            for days_out in PACE_INTERVALS:
+                snapshot_date = current - timedelta(days=days_out)
+
+                # Build query based on pace_type
+                if pace_type == 'total':
+                    # All valid bookings
+                    result = db.execute(
+                        text("""
+                            SELECT COALESCE(SUM(covers), 0) as total_covers
+                            FROM resos_bookings_data
+                            WHERE booking_date = :target_date
+                            AND status IN :valid_statuses
+                            AND booking_placed <= :snapshot_date
+                        """),
+                        {
+                            "target_date": current,
+                            "valid_statuses": VALID_STATUSES,
+                            "snapshot_date": snapshot_date
+                        }
+                    )
+                elif pace_type == 'resident':
+                    # Hotel guests only
+                    result = db.execute(
+                        text("""
+                            SELECT COALESCE(SUM(covers), 0) as total_covers
+                            FROM resos_bookings_data
+                            WHERE booking_date = :target_date
+                            AND status IN :valid_statuses
+                            AND booking_placed <= :snapshot_date
+                            AND is_hotel_guest = true
+                        """),
+                        {
+                            "target_date": current,
+                            "valid_statuses": VALID_STATUSES,
+                            "snapshot_date": snapshot_date
+                        }
+                    )
+                else:  # non_resident
+                    # Non-hotel guests
+                    result = db.execute(
+                        text("""
+                            SELECT COALESCE(SUM(covers), 0) as total_covers
+                            FROM resos_bookings_data
+                            WHERE booking_date = :target_date
+                            AND status IN :valid_statuses
+                            AND booking_placed <= :snapshot_date
+                            AND (is_hotel_guest = false OR is_hotel_guest IS NULL)
+                        """),
+                        {
+                            "target_date": current,
+                            "valid_statuses": VALID_STATUSES,
+                            "snapshot_date": snapshot_date
+                        }
+                    )
+
+                row = result.fetchone()
+                pace_values[f"d{days_out}"] = row.total_covers if row else 0
+
+            # Upsert pace record
+            columns = ", ".join(pace_values.keys())
+            placeholders = ", ".join([f":{k}" for k in pace_values.keys()])
+            updates = ", ".join([f"{k} = :{k}" for k in pace_values.keys()])
+
+            db.execute(
+                text(f"""
+                    INSERT INTO resos_booking_pace (booking_date, pace_type, {columns}, updated_at)
+                    VALUES (:booking_date, :pace_type, {placeholders}, NOW())
+                    ON CONFLICT (booking_date, pace_type) DO UPDATE SET
+                        {updates},
+                        updated_at = NOW()
+                """),
+                {"booking_date": current, "pace_type": pace_type, **pace_values}
+            )
+
+        current += timedelta(days=1)
+
+    db.commit()
+    logger.info("Resos booking pace table updated (3 types: total, resident, non_resident)")

backend/jobs/resos_bookings_sync.py

@@ -0,0 +1,397 @@
+"""
+Resos Bookings Data Sync Job
+Syncs restaurant bookings to resos_bookings_data table
+Pattern: Replicates newbook bookings sync but adapted for Resos covers/stats
+"""
+import json
+import logging
+import base64
+from datetime import date, datetime, timedelta
+from typing import Optional, Tuple, List, Dict, Any
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+from services.resos_client import ResosClient
+
+logger = logging.getLogger(__name__)
+
+# Valid booking statuses for aggregation
+# Note: Resos uses 'approved' for confirmed future reservations, 'left' for completed meals
+VALID_STATUSES = ('approved', 'arrived', 'seated', 'left')
+
+
+def get_config_value(db, key: str) -> Optional[str]:
+    """Get a configuration value from system_config table."""
+    result = db.execute(
+        text("SELECT config_value, is_encrypted FROM system_config WHERE config_key = :key"),
+        {"key": key}
+    )
+    row = result.fetchone()
+    if not row or not row.config_value:
+        return None
+
+    # Decrypt if encrypted
+    if row.is_encrypted:
+        try:
+            return base64.b64decode(row.config_value.encode()).decode()
+        except Exception as e:
+            logger.warning(f"Failed to decrypt {key}: {e}")
+            return row.config_value
+
+    return row.config_value
+
+
+def load_resos_custom_field_mappings(db) -> Dict[str, Dict[str, Any]]:
+    """
+    Load custom field mappings from resos_custom_field_mapping table.
+    Returns dict: {field_id: {"maps_to": "hotel_guest", "value_for_true": "Yes"}}
+    """
+    result = db.execute(text("""
+        SELECT field_id, maps_to, value_for_true
+        FROM resos_custom_field_mapping
+        WHERE maps_to != 'ignore'
+    """))
+    mappings = {}
+    for row in result.fetchall():
+        mappings[row.field_id] = {
+            "maps_to": row.maps_to,
+            "value_for_true": row.value_for_true
+        }
+    logger.info(f"Loaded {len(mappings)} custom field mappings")
+    return mappings
+
+
+def load_resos_opening_hours_mappings(db) -> Dict[str, Dict[str, str]]:
+    """
+    Load opening hours mappings from resos_opening_hours_mapping table.
+    Returns dict: {opening_hour_id: {"period_type": "dinner", "display_name": "..."}}
+    """
+    result = db.execute(text("""
+        SELECT opening_hour_id, period_type, display_name
+        FROM resos_opening_hours_mapping
+        WHERE period_type != 'ignore'
+    """))
+    mappings = {}
+    for row in result.fetchall():
+        mappings[row.opening_hour_id] = {
+            "period_type": row.period_type,
+            "display_name": row.display_name
+        }
+    logger.info(f"Loaded {len(mappings)} opening hours mappings")
+    return mappings
+
+
+def parse_group_exclude_field(group_exclude_field: Optional[str], primary_booking_number: Optional[str]) -> Tuple[List[str], List[str]]:
+    """
+    Parse group_exclude_field to extract linked bookings and exclude markers.
+
+    Args:
+        group_exclude_field: Raw field like "#12346,#12347,NOT-#56748"
+        primary_booking_number: Primary booking from hotel_booking_number field
+
+    Returns:
+        (all_booking_numbers, exclude_numbers)
+
+    Example:
+        Input: "#12346,#12347,NOT-#56748", "NB12345"
+        Returns: (["NB12345", "NB12346", "NB12347"], ["NB56748"])
+    """
+    all_booking_numbers = []
+    exclude_numbers = []
+
+    # Always include primary booking number
+    if primary_booking_number:
+        all_booking_numbers.append(primary_booking_number)
+
+    if not group_exclude_field:
+        return all_booking_numbers, exclude_numbers
+
+    # Parse comma-separated entries
+    parts = group_exclude_field.split(',')
+    for part in parts:
+        part = part.strip()
+
+        if part.upper().startswith('NOT-#'):
+            # Exclude marker: NOT-#56748 → NB56748
+            booking_num = part[5:]  # Remove "NOT-#"
+            exclude_numbers.append(f"NB{booking_num}")
+
+        elif part.startswith('#'):
+            # Additional booking: #12346 → NB12346
+            booking_num = part[1:]  # Remove "#"
+            all_booking_numbers.append(f"NB{booking_num}")
+
+    return all_booking_numbers, exclude_numbers
+
+
+def extract_custom_field_value(custom_fields: List[Dict[str, Any]], field_id: str, cf_mappings: Dict[str, Dict[str, Any]]) -> Optional[Any]:
+    """
+    Extract value from Resos custom fields array.
+
+    Args:
+        custom_fields: Array of custom field objects from Resos API
+        field_id: The field ID to look for
+        cf_mappings: Mapping configuration
+
+    Returns:
+        Extracted value (boolean for hotel_guest/dbb/package, string for booking_number, etc.)
+    """
+    if field_id not in cf_mappings:
+        return None
+
+    mapping = cf_mappings[field_id]
+    maps_to = mapping["maps_to"]
+    value_for_true = mapping.get("value_for_true")
+
+    # Find the field in custom_fields array
+    field_value = None
+    field_value_label = None
+    for cf in custom_fields:
+        cf_id = cf.get("id") or cf.get("_id") or cf.get("fieldId")
+        if cf_id == field_id:
+            field_value = cf.get("value")
+            field_value_label = cf.get("multipleChoiceValueName") or cf.get("value")
+            break
+
+    if field_value is None and field_value_label is None:
+        return None
+
+    # For boolean mappings (hotel_guest, dbb, package)
+    if maps_to in ("hotel_guest", "dbb", "package"):
+        if value_for_true:
+            return str(field_value_label) == str(value_for_true)
+        else:
+            # Auto-detect: check for "yes", "true", "1"
+            return str(field_value_label).lower() in ("yes", "true", "1")
+
+    # For string mappings (booking_number, group_exclude, allergies)
+    return str(field_value) if field_value else str(field_value_label) if field_value_label else None
+
+
+async def sync_resos_bookings_data(
+    from_date: date,
+    to_date: date,
+    triggered_by: str = "scheduler"
+):
+    """
+    Sync Resos bookings to resos_bookings_data.
+
+    Date range: Historical -365 days + Forecast +365 days (daily sync)
+    PII handling: Remove guest details, store only aggregate covers
+    """
+    logger.info(f"Starting Resos bookings sync from {from_date} to {to_date} (triggered by {triggered_by})")
+
+    db = SyncSessionLocal()
+
+    try:
+        # Log sync start
+        db.execute(
+            text("""
+            INSERT INTO sync_log (sync_type, source, started_at, status, date_from, date_to, triggered_by)
+            VALUES ('bookings_data', 'resos', NOW(), 'running', :from_date, :to_date, :triggered_by)
+            """),
+            {"from_date": from_date, "to_date": to_date, "triggered_by": triggered_by}
+        )
+        db.commit()
+
+        # Load Resos API key
+        api_key = get_config_value(db, 'resos_api_key')
+
+        if not api_key:
+            raise Exception("Resos API key not configured")
+
+        # Load mappings
+        cf_mappings = load_resos_custom_field_mappings(db)
+        oh_mappings = load_resos_opening_hours_mappings(db)
+
+        async with ResosClient(api_key=api_key) as client:
+            # Test connection
+            if not await client.test_connection():
+                raise Exception("Resos connection failed")
+
+            # Fetch bookings
+            logger.info(f"Fetching bookings from Resos API for {from_date} to {to_date}")
+            bookings = await client.get_bookings(from_date, to_date)
+            logger.info(f"Fetched {len(bookings)} bookings from Resos")
+
+            records_created = 0
+            records_updated = 0
+
+            for booking in bookings:
+                resos_id = booking.get("_id")
+                if not resos_id:
+                    continue
+
+                booking_date_str = booking.get("date")
+                booking_date_obj = date.fromisoformat(booking_date_str) if booking_date_str else None
+
+                if not booking_date_obj:
+                    continue
+
+                # Extract opening hour ID and map to period type
+                opening_hour_id = booking.get("openingHourId")
+                period_type = None
+                if opening_hour_id and opening_hour_id in oh_mappings:
+                    period_type = oh_mappings[opening_hour_id]["period_type"]
+
+                # Extract custom fields using mappings
+                custom_fields = booking.get("customFields", [])
+
+                is_hotel_guest = None
+                is_dbb = None
+                is_package = None
+                hotel_booking_number = None
+                group_exclude_field = None
+
+                for field_id, mapping in cf_mappings.items():
+                    maps_to = mapping["maps_to"]
+
+                    if maps_to == "hotel_guest":
+                        is_hotel_guest = extract_custom_field_value(custom_fields, field_id, cf_mappings)
+                    elif maps_to == "dbb":
+                        is_dbb = extract_custom_field_value(custom_fields, field_id, cf_mappings)
+                    elif maps_to == "package":
+                        is_package = extract_custom_field_value(custom_fields, field_id, cf_mappings)
+                    elif maps_to == "booking_number":
+                        hotel_booking_number = extract_custom_field_value(custom_fields, field_id, cf_mappings)
+                    elif maps_to == "group_exclude":
+                        group_exclude_field = extract_custom_field_value(custom_fields, field_id, cf_mappings)
+
+                # Remove PII from raw JSON (remove guest object)
+                raw_booking = {k: v for k, v in booking.items() if k != "guest"}
+                raw_json_str = json.dumps(raw_booking)
+
+                # Extract other booking details
+                covers = booking.get("people", 0)
+                status = booking.get("status")
+                source = booking.get("source")
+                booking_time_str = booking.get("time")
+
+                # Parse table information
+                tables = booking.get("tables", [])
+                table_name = tables[0].get("name") if tables else None
+                table_area = None
+                if tables and tables[0].get("area"):
+                    table_area = tables[0]["area"].get("name")
+
+                # Parse booking placed timestamp
+                booking_placed_str = booking.get("createdAt")
+                booking_placed = None
+                if booking_placed_str:
+                    try:
+                        booking_placed = datetime.fromisoformat(booking_placed_str.replace('Z', '+00:00'))
+                    except:
+                        pass
+
+                # Parse notes (sanitized - no PII)
+                notes_array = booking.get("restaurantNotes", [])
+                notes = ', '.join([str(note) for note in notes_array]) if notes_array else None
+
+                # Check if record exists
+                existing = db.execute(
+                    text("SELECT id FROM resos_bookings_data WHERE resos_id = :rid"),
+                    {"rid": resos_id}
+                ).fetchone()
+
+                # Upsert booking
+                db.execute(
+                    text("""
+                    INSERT INTO resos_bookings_data (
+                        resos_id, booking_date, booking_time, opening_hour_id, period_type,
+                        covers, status, source, table_name, table_area,
+                        is_hotel_guest, is_dbb, is_package, hotel_booking_number, group_exclude_field,
+                        total_guests, booking_placed, notes, raw_json, fetched_at
+                    ) VALUES (
+                        :resos_id, :booking_date, :booking_time, :opening_hour_id, :period_type,
+                        :covers, :status, :source, :table_name, :table_area,
+                        :is_hotel_guest, :is_dbb, :is_package, :hotel_booking_number, :group_exclude_field,
+                        :total_guests, :booking_placed, :notes, :raw_json, NOW()
+                    )
+                    ON CONFLICT (resos_id) DO UPDATE SET
+                        status = :status,
+                        covers = :covers,
+                        period_type = :period_type,
+                        is_hotel_guest = COALESCE(:is_hotel_guest, resos_bookings_data.is_hotel_guest),
+                        is_dbb = COALESCE(:is_dbb, resos_bookings_data.is_dbb),
+                        is_package = COALESCE(:is_package, resos_bookings_data.is_package),
+                        hotel_booking_number = COALESCE(:hotel_booking_number, resos_bookings_data.hotel_booking_number),
+                        group_exclude_field = COALESCE(:group_exclude_field, resos_bookings_data.group_exclude_field),
+                        raw_json = :raw_json,
+                        fetched_at = NOW()
+                    """),
+                    {
+                        "resos_id": resos_id,
+                        "booking_date": booking_date_obj,
+                        "booking_time": booking_time_str,
+                        "opening_hour_id": opening_hour_id,
+                        "period_type": period_type,
+                        "covers": covers,
+                        "status": status,
+                        "source": source,
+                        "table_name": table_name,
+                        "table_area": table_area,
+                        "is_hotel_guest": is_hotel_guest,
+                        "is_dbb": is_dbb,
+                        "is_package": is_package,
+                        "hotel_booking_number": hotel_booking_number,
+                        "group_exclude_field": group_exclude_field,
+                        "total_guests": covers,  # Same as covers for restaurants
+                        "booking_placed": booking_placed,
+                        "notes": notes,
+                        "raw_json": raw_json_str
+                    }
+                )
+
+                if existing:
+                    records_updated += 1
+                else:
+                    records_created += 1
+
+                db.commit()
+
+            # Update sync log
+            db.execute(
+                text("""
+                UPDATE sync_log
+                SET completed_at = NOW(), status = 'success',
+                    records_fetched = :fetched, records_created = :created, records_updated = :updated
+                WHERE id = (
+                    SELECT id FROM sync_log
+                    WHERE source = 'resos' AND sync_type = 'bookings_data' AND status = 'running'
+                    ORDER BY started_at DESC LIMIT 1
+                )
+                """),
+                {"fetched": len(bookings), "created": records_created, "updated": records_updated}
+            )
+            db.commit()
+
+            logger.info(f"Resos bookings sync completed: {records_created} created, {records_updated} updated")
+
+            # Trigger aggregation
+            logger.info("Triggering Resos bookings aggregation...")
+            try:
+                import asyncio
+                from jobs.resos_aggregation import aggregate_resos_bookings
+                await aggregate_resos_bookings(triggered_by=triggered_by)
+                logger.info("Resos bookings aggregation completed")
+            except Exception as agg_error:
+                logger.warning(f"Resos aggregation failed (non-fatal): {agg_error}")
+
+    except Exception as e:
+        logger.error(f"Resos bookings sync failed: {e}", exc_info=True)
+        db.execute(
+            text("""
+            UPDATE sync_log
+            SET completed_at = NOW(), status = 'failed', error_message = :error
+            WHERE id = (
+                SELECT id FROM sync_log
+                WHERE source = 'resos' AND sync_type = 'bookings_data' AND status = 'running'
+                ORDER BY started_at DESC LIMIT 1
+            )
+            """),
+            {"error": str(e)[:500]}
+        )
+        db.commit()
+        raise
+    finally:
+        db.close()

backend/jobs/revenue_aggregation.py

@@ -0,0 +1,153 @@
+"""
+Revenue aggregation - consolidates earned revenue by department
+"""
+import logging
+from datetime import datetime
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+def get_config_value(db, key: str) -> str | None:
+    """Get a config value from system_config"""
+    result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = :key"),
+        {"key": key}
+    )
+    row = result.fetchone()
+    return row.config_value if row else None
+
+
+def set_config_value(db, key: str, value: str):
+    """Set a config value in system_config"""
+    db.execute(
+        text("""
+            INSERT INTO system_config (config_key, config_value, updated_at)
+            VALUES (:key, :value, NOW())
+            ON CONFLICT (config_key) DO UPDATE SET
+                config_value = :value,
+                updated_at = NOW()
+        """),
+        {"key": key, "value": value}
+    )
+
+
+async def aggregate_revenue(since_timestamp: str = None):
+    """
+    Aggregate earned revenue data by department into newbook_net_revenue_data.
+
+    - Joins newbook_earned_revenue_data with newbook_gl_accounts to get department
+    - Sums net amounts by date and department
+    - Only processes dates with data fetched since last aggregation (or all if first run)
+
+    Args:
+        since_timestamp: Optional timestamp to process data from (ISO format)
+                        If not provided, uses last_revenue_aggregation_at config
+    """
+    logger.info("Starting revenue aggregation...")
+
+    db = SyncSessionLocal()
+    try:
+        # Get last aggregation time if not provided
+        if since_timestamp is None:
+            since_timestamp = get_config_value(db, 'last_revenue_aggregation_at')
+
+        # Find dates with new/updated data
+        if since_timestamp:
+            logger.info(f"Aggregating revenue data updated since {since_timestamp}")
+            result = db.execute(
+                text("""
+                    SELECT DISTINCT date
+                    FROM newbook_earned_revenue_data
+                    WHERE fetched_at > :since
+                    ORDER BY date
+                """),
+                {"since": since_timestamp}
+            )
+        else:
+            logger.info("Aggregating all revenue data (first run)")
+            result = db.execute(
+                text("""
+                    SELECT DISTINCT date
+                    FROM newbook_earned_revenue_data
+                    ORDER BY date
+                """)
+            )
+
+        dates_to_process = [row.date for row in result.fetchall()]
+
+        if not dates_to_process:
+            logger.info("No new revenue data to aggregate")
+            return {"dates_processed": 0, "message": "No new data"}
+
+        logger.info(f"Found {len(dates_to_process)} dates to aggregate")
+
+        # Aggregate each date
+        for target_date in dates_to_process:
+            # Get totals by department for this date
+            result = db.execute(
+                text("""
+                    SELECT
+                        COALESCE(g.department, 'other') as department,
+                        SUM(e.amount_net) as total_net
+                    FROM newbook_earned_revenue_data e
+                    LEFT JOIN newbook_gl_accounts g ON e.gl_code = g.gl_code
+                    WHERE e.date = :date
+                    GROUP BY g.department
+                """),
+                {"date": target_date}
+            )
+
+            totals = {"accommodation": 0, "dry": 0, "wet": 0}
+            for row in result.fetchall():
+                if row.department in totals:
+                    totals[row.department] = float(row.total_net or 0)
+
+            # Upsert into newbook_net_revenue_data
+            db.execute(
+                text("""
+                    INSERT INTO newbook_net_revenue_data (date, accommodation, dry, wet, aggregated_at)
+                    VALUES (:date, :accommodation, :dry, :wet, NOW())
+                    ON CONFLICT (date) DO UPDATE SET
+                        accommodation = :accommodation,
+                        dry = :dry,
+                        wet = :wet,
+                        aggregated_at = NOW()
+                """),
+                {
+                    "date": target_date,
+                    "accommodation": round(totals["accommodation"], 2),
+                    "dry": round(totals["dry"], 2),
+                    "wet": round(totals["wet"], 2)
+                }
+            )
+
+        db.commit()
+
+        # Update last aggregation timestamp
+        set_config_value(db, 'last_revenue_aggregation_at', datetime.now().isoformat())
+        db.commit()
+
+        logger.info(f"Revenue aggregation complete: {len(dates_to_process)} dates")
+        return {
+            "dates_processed": len(dates_to_process),
+            "message": f"Aggregated {len(dates_to_process)} dates"
+        }
+
+    except Exception as e:
+        logger.error(f"Revenue aggregation failed: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+async def backfill_revenue_aggregation():
+    """
+    Backfill all historical revenue data.
+    Forces re-aggregation of all dates regardless of last run time.
+    """
+    logger.info("Starting revenue backfill aggregation...")
+    # Pass epoch time to force processing all data
+    return await aggregate_revenue(since_timestamp="1970-01-01T00:00:00")

backend/jobs/scrape_booking_rates.py

@@ -0,0 +1,165 @@
+"""
+Scheduled Booking.com Rate Scraping Job
+
+Priority-based scheduling for 365-day coverage (all queued daily):
+- High (priority 10): next 30 days
+- Medium (priority 5): days 31-180
+- Low (priority 2): days 181-365
+
+Queue processes in priority order. If rate-limited/blocked, lower priority
+dates remain queued for the next run.
+
+Uses a queue-based approach:
+1. Populate the queue with dates and priorities
+2. Process the queue in priority order
+3. Failed dates are retried (up to 3 attempts)
+4. On blocking, the queue pauses and resumes after cooldown
+
+Schedule: Daily at configurable time (default 05:30)
+"""
+import asyncio
+import logging
+from datetime import date, timedelta
+
+from sqlalchemy import text
+from database import SyncSessionLocal
+from services.booking_scraper import (
+    populate_queue,
+    process_queue,
+    clear_old_queue_items,
+    cleanup_stale_batches,
+    get_scrape_config,
+)
+
+logger = logging.getLogger(__name__)
+
+# Priority levels (higher = processed first)
+PRIORITY_HIGH = 10    # 0-30 days
+PRIORITY_MEDIUM = 5   # 31-180 days
+PRIORITY_LOW = 2      # 181-365 days
+
+
+def get_high_priority_dates() -> list[date]:
+    """High priority: today + 30 days."""
+    today = date.today()
+    return [today + timedelta(days=i) for i in range(31)]
+
+
+def get_medium_priority_dates() -> list[date]:
+    """Medium priority: days 31-180."""
+    today = date.today()
+    return [today + timedelta(days=i) for i in range(31, 181)]
+
+
+def get_low_priority_dates() -> list[date]:
+    """Low priority: days 181-365."""
+    today = date.today()
+    return [today + timedelta(days=i) for i in range(181, 366)]
+
+
+def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
+    """
+    For a target date, determine its priority tier and when it will next be scraped.
+
+    Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'.
+    All dates are queued daily, so next scrape is always today (or tomorrow if
+    today's run has passed).
+    """
+    today = date.today()
+    offset = (target_date - today).days
+
+    if offset < 0:
+        return ('none', None)
+    if offset > 365:
+        return ('none', None)
+
+    # All tiers run daily - next scrape is today
+    if offset <= 30:
+        return ('high', today)
+    elif offset <= 180:
+        return ('medium', today)
+    else:
+        return ('low', today)
+
+
+def run_scheduled_booking_scrape():
+    """
+    Main scheduled job: populate queue with today's dates, then process.
+    """
+    db = SyncSessionLocal()
+    try:
+        # Check if scraper is enabled
+        result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
+        ).fetchone()
+        if not result or result.config_value != 'true':
+            logger.debug("Scheduled booking scrape skipped (disabled)")
+            return
+
+        if not get_scrape_config(db):
+            logger.warning("Scheduled booking scrape skipped (no location configured)")
+            return
+
+        # Clean up stale running batches and old queue items
+        cleanup_stale_batches(db, max_age_minutes=120)
+        clear_old_queue_items(db, days=3)
+
+        # Gather dates with priorities
+        high = get_high_priority_dates()
+        medium = get_medium_priority_dates()
+        low = get_low_priority_dates()
+
+        priorities = {}
+        for d in high:
+            priorities[d] = PRIORITY_HIGH
+        for d in medium:
+            priorities[d] = max(priorities.get(d, 0), PRIORITY_MEDIUM)
+        for d in low:
+            priorities[d] = max(priorities.get(d, 0), PRIORITY_LOW)
+
+        all_dates = sorted(priorities.keys())
+
+        if not all_dates:
+            logger.info("Scheduled booking scrape: no dates to scrape today")
+            return
+
+        logger.info(
+            f"Scheduled booking scrape: queuing {len(all_dates)} dates "
+            f"(high={len(high)}, medium={len(medium)}, low={len(low)})"
+        )
+
+        # Populate queue
+        populate_queue(db, all_dates, priorities)
+
+        # Process queue
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            result = loop.run_until_complete(process_queue(db))
+            if result.get('success'):
+                logger.info(
+                    f"Scheduled booking scrape completed: "
+                    f"{result.get('dates_completed', 0)} dates, "
+                    f"{result.get('rates_scraped', 0)} rates"
+                )
+            elif result.get('blocked'):
+                logger.warning(
+                    f"Scheduled booking scrape blocked: {result.get('block_reason')}. "
+                    f"Completed {result.get('dates_completed', 0)} dates. "
+                    f"Remaining dates stay queued for retry."
+                )
+            else:
+                logger.error(f"Scheduled booking scrape failed: {result.get('error')}")
+        finally:
+            loop.close()
+
+    except Exception as e:
+        logger.error(f"Scheduled booking scrape error: {e}", exc_info=True)
+    finally:
+        db.close()
+
+
+async def run_scheduled_booking_scrape_async():
+    """Async wrapper for APScheduler."""
+    loop = asyncio.get_event_loop()
+    await loop.run_in_executor(None, run_scheduled_booking_scrape)

backend/jobs/weekly_forecast_snapshot.py

@@ -0,0 +1,197 @@
+"""
+Weekly Forecast Snapshot Job
+Automatically creates blended forecast snapshots using MAPE-weighted model blending with 60/40 budget/prior year blend.
+Uses the blended_tuned_weighted service for accuracy-optimized forecasts.
+"""
+import logging
+import uuid
+from datetime import date, datetime, timedelta
+from sqlalchemy import text
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+def get_config_value(db, key: str, default: str = None) -> str:
+    """Get a config value from system_config"""
+    try:
+        result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = :key"),
+            {"key": key}
+        )
+        row = result.fetchone()
+        if row and row.config_value:
+            return row.config_value
+        return default
+    except Exception as e:
+        logger.error(f"Error getting config {key}: {e}")
+        return default
+
+
+def is_forecast_snapshot_enabled(db) -> bool:
+    """Check if automated forecast snapshots are enabled"""
+    value = get_config_value(db, "forecast_snapshot_enabled")
+    if value:
+        return value.lower() in ('true', '1', 'yes', 'enabled')
+    return False
+
+
+def get_forecast_snapshot_days_ahead(db) -> int:
+    """Get number of days ahead to forecast"""
+    days_str = get_config_value(db, "forecast_snapshot_days_ahead", "90")
+    try:
+        return int(days_str)
+    except ValueError:
+        return 90
+
+
+async def run_weekly_forecast_snapshot():
+    """
+    Run weekly blended forecast snapshot using MAPE-weighted + 60/40 blend.
+    This is the single source of truth for forecast snapshots.
+    - Stage 1: MAPE-weighted blend of Prophet, XGBoost, CatBoost (+ Pickup for pace metrics)
+    - Stage 2: 60% model blend + 40% budget (revenue) or prior year (non-revenue)
+    """
+    db = SyncSessionLocal()
+    run_id = str(uuid.uuid4())
+
+    try:
+        # Check if enabled
+        if not is_forecast_snapshot_enabled(db):
+            logger.info("Weekly forecast snapshot is disabled, skipping")
+            return
+
+        days_ahead = get_forecast_snapshot_days_ahead(db)
+        forecast_from = date.today()
+        forecast_to = date.today() + timedelta(days=days_ahead)
+
+        logger.info(f"Starting weekly blended forecast snapshot: {forecast_from} to {forecast_to}")
+
+        # Log run start
+        db.execute(
+            text("""
+            INSERT INTO forecast_runs (
+                run_id, run_type, started_at, status,
+                forecast_from, forecast_to, models_run, triggered_by
+            ) VALUES (
+                :run_id, 'scheduled', NOW(), 'running',
+                :forecast_from, :forecast_to, :models, :triggered_by
+            )
+            """),
+            {
+                "run_id": run_id,
+                "forecast_from": forecast_from,
+                "forecast_to": forecast_to,
+                "models": '["blended"]',
+                "triggered_by": "forecast_snapshot"
+            }
+        )
+        db.commit()
+
+        # Get active metrics
+        result = db.execute(
+            text("""
+                SELECT metric_code, metric_name
+                FROM forecast_metrics
+                WHERE is_active = TRUE
+            """)
+        )
+        metrics = result.fetchall()
+
+        # Import MAPE-weighted blended model with 60/40 budget blend
+        from services.forecasting.blended_tuned_weighted import run_blended_tuned_weighted_forecast
+
+        # Run blended forecast for each metric
+        total_forecasts = 0
+        for metric in metrics:
+            metric_code = metric.metric_code
+            try:
+                logger.info(f"Generating MAPE-weighted + 60/40 blended forecast for {metric_code}")
+                forecasts = await run_blended_tuned_weighted_forecast(
+                    db=db,
+                    metric_code=metric_code,
+                    start_date=forecast_from,
+                    end_date=forecast_to,
+                    save_to_db=True,
+                    run_id=run_id
+                    # apply_60_40_blend defaults to True
+                )
+                total_forecasts += len(forecasts)
+                logger.info(f"Generated {len(forecasts)} MAPE-weighted + 60/40 forecasts for {metric_code}")
+            except Exception as e:
+                logger.error(f"Blended forecast failed for {metric_code}: {e}")
+                db.rollback()
+                continue
+
+        # Update run status to success
+        db.execute(
+            text("""
+            UPDATE forecast_runs
+            SET completed_at = NOW(), status = 'success'
+            WHERE run_id = :run_id
+            """),
+            {"run_id": run_id}
+        )
+        db.commit()
+
+        # Log completion to sync_log
+        db.execute(
+            text("""
+                INSERT INTO sync_log (sync_type, source, started_at, completed_at, status, records_created, triggered_by)
+                VALUES (:sync_type, :source, :started_at, :completed_at, :status, :records_created, :triggered_by)
+            """),
+            {
+                "sync_type": "forecast_snapshot",
+                "source": "blended_tuned_weighted",
+                "started_at": datetime.now(),
+                "completed_at": datetime.now(),
+                "status": "success",
+                "records_created": total_forecasts,
+                "triggered_by": "scheduler"
+            }
+        )
+        db.commit()
+
+        logger.info(f"Weekly forecast snapshot completed: {total_forecasts} total forecasts generated")
+
+    except Exception as e:
+        logger.error(f"Weekly forecast snapshot failed: {e}")
+
+        # Rollback and update run status
+        db.rollback()
+        try:
+            db.execute(
+                text("""
+                UPDATE forecast_runs
+                SET completed_at = NOW(), status = 'failed', error_message = :error
+                WHERE run_id = :run_id
+                """),
+                {"run_id": run_id, "error": str(e)}
+            )
+            db.commit()
+        except:
+            pass
+
+        # Log error to sync_log
+        try:
+            db.execute(
+                text("""
+                    INSERT INTO sync_log (sync_type, source, started_at, completed_at, status, error_message, triggered_by)
+                    VALUES (:sync_type, :source, :started_at, :completed_at, :status, :error_message, :triggered_by)
+                """),
+                {
+                    "sync_type": "forecast_snapshot",
+                    "source": "blended_tuned_weighted",
+                    "started_at": datetime.now(),
+                    "completed_at": datetime.now(),
+                    "status": "error",
+                    "error_message": str(e),
+                    "triggered_by": "scheduler"
+                }
+            )
+            db.commit()
+        except:
+            pass
+        raise
+    finally:
+        db.close()

backend/main.py

@@ -0,0 +1,138 @@
+"""
+Forecasting Application — FastAPI Backend
+Auth is handled by the central HNF stack cookie (hnf_session).
+"""
+import logging
+import sys
+from contextlib import asynccontextmanager
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler(sys.stdout)]
+)
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from sqlalchemy import text
+
+from database import async_engine
+from api import (
+    forecast, sync, export, budget, accuracy, evolution, crossref,
+    explain, config, historical, resos, backtest, sync_bookings,
+    resos_sync, reports, special_dates, backup, public, bookability,
+    competitor_rates, ai_insights,
+)
+from scheduler import start_scheduler, shutdown_scheduler
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Apply database schema (idempotent CREATE TABLE IF NOT EXISTS)
+    try:
+        import os
+        from database import SyncSessionLocal
+        schema_path = os.path.join(os.path.dirname(__file__), 'schema.sql')
+        if os.path.exists(schema_path):
+            db = SyncSessionLocal()
+            try:
+                with open(schema_path) as f:
+                    sql = f.read()
+                db.execute(text(sql))
+                db.commit()
+                logging.getLogger(__name__).info("Schema applied successfully")
+            except Exception as e:
+                logging.getLogger(__name__).warning(f"Schema init failed: {e}")
+                db.rollback()
+            finally:
+                db.close()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Schema load failed: {e}")
+
+    # Startup: clean up stale scrape batches
+    try:
+        from services.booking_scraper import cleanup_stale_batches
+        from database import SyncSessionLocal
+        db = SyncSessionLocal()
+        try:
+            cleanup_stale_batches(db, max_age_minutes=10)
+        finally:
+            db.close()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Stale batch cleanup failed: {e}")
+
+    # Ensure ai_insights table exists
+    try:
+        from database import SyncSessionLocal
+        db = SyncSessionLocal()
+        try:
+            db.execute(text("""
+                CREATE TABLE IF NOT EXISTS ai_insights (
+                    id SERIAL PRIMARY KEY,
+                    generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                    insight_type VARCHAR(50) NOT NULL DEFAULT 'daily_summary',
+                    content TEXT NOT NULL,
+                    model VARCHAR(100),
+                    input_tokens INTEGER,
+                    output_tokens INTEGER,
+                    data_snapshot JSONB,
+                    triggered_by VARCHAR(50) DEFAULT 'scheduler'
+                )
+            """))
+            db.execute(text("""
+                CREATE INDEX IF NOT EXISTS idx_ai_insights_generated
+                ON ai_insights(generated_at DESC)
+            """))
+            db.commit()
+        finally:
+            db.close()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"AI insights table creation failed: {e}")
+
+    start_scheduler()
+    yield
+    shutdown_scheduler()
+
+
+app = FastAPI(
+    title="Forecasting API",
+    description="Hotel & Restaurant Forecasting Service",
+    version="2.0.0",
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# All routes — nginx strips /forecasting/api/ prefix before reaching here
+app.include_router(forecast.router,          prefix="/forecast",          tags=["Forecasts"])
+app.include_router(sync.router,              prefix="/sync",              tags=["Data Sync"])
+app.include_router(export.router,            prefix="/export",            tags=["Exports"])
+app.include_router(budget.router,            prefix="/budget",            tags=["Budgets"])
+app.include_router(accuracy.router,          prefix="/accuracy",          tags=["Accuracy"])
+app.include_router(evolution.router,         prefix="/evolution",         tags=["Forecast Evolution"])
+app.include_router(crossref.router,          prefix="/crossref",          tags=["Cross-Reference"])
+app.include_router(explain.router,           prefix="/explain",           tags=["Explainability"])
+app.include_router(config.router,            prefix="/config",            tags=["Configuration"])
+app.include_router(historical.router,        prefix="/historical",        tags=["Historical Data"])
+app.include_router(resos.router,             prefix="/resos",             tags=["Resos Mapping"])
+app.include_router(backtest.router,          prefix="/backtest",          tags=["Backtesting"])
+app.include_router(sync_bookings.router,     prefix="/sync",              tags=["Data Sync"])
+app.include_router(resos_sync.router,        prefix="/sync",              tags=["Data Sync"])
+app.include_router(reports.router,           prefix="/reports",           tags=["Reports"])
+app.include_router(special_dates.router,     prefix="/settings",          tags=["Settings"])
+app.include_router(backup.router,            prefix="/backup",            tags=["Backup & Restore"])
+app.include_router(public.router,            prefix="/public",            tags=["Public API"])
+app.include_router(bookability.router,       prefix="/bookability",       tags=["Bookability"])
+app.include_router(competitor_rates.router,  prefix="/competitor-rates",  tags=["Competitor Rates"])
+app.include_router(ai_insights.router,       prefix="/ai-insights",       tags=["AI Insights"])
+
+
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy", "service": "forecasting-api"}

backend/requirements.txt

@@ -0,0 +1,24 @@
+fastapi==0.109.0
+uvicorn[standard]==0.27.0
+sqlalchemy==2.0.25
+asyncpg==0.29.0
+psycopg2-binary==2.9.9
+python-jose[cryptography]==3.3.0
+passlib[bcrypt]==1.7.4
+python-multipart==0.0.6
+httpx==0.26.0
+pandas==2.2.0
+numpy==1.26.3
+prophet==1.1.4
+xgboost==2.0.3
+catboost==1.2.7
+shap==0.44.1
+scikit-learn==1.4.0
+apscheduler==3.10.4
+openpyxl==3.1.2
+pydantic==2.5.3
+pydantic-settings==2.1.0
+python-dotenv==1.0.0
+python-dateutil==2.8.2
+playwright>=1.40.0
+anthropic>=0.42.0

backend/scheduler.py

@@ -0,0 +1,366 @@
+"""
+APScheduler configuration for scheduled jobs
+"""
+import logging
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.cron import CronTrigger
+from sqlalchemy import text
+
+from jobs.data_sync import (
+    sync_newbook_data,
+    sync_resos_data,
+    sync_newbook_occupancy_report,
+    sync_newbook_earned_revenue
+)
+from jobs.resos_bookings_sync import sync_resos_bookings_data
+from api.sync_bookings import run_bookings_data_sync
+from jobs.aggregation import run_aggregation
+from jobs.forecast_daily import run_daily_forecast
+from jobs.pickup_snapshot import run_pickup_snapshot
+from jobs.pace_snapshot_v2 import run_pace_snapshot_v2
+from jobs.accuracy_calc import run_accuracy_calculation
+from jobs.weekly_forecast_snapshot import run_weekly_forecast_snapshot
+from jobs.fetch_current_rates import run_fetch_current_rates
+from jobs.scrape_booking_rates import run_scheduled_booking_scrape_async
+from jobs.ai_insights import run_ai_insights_generation
+from database import SyncSessionLocal
+
+logger = logging.getLogger(__name__)
+
+scheduler = AsyncIOScheduler(
+    job_defaults={
+        'misfire_grace_time': 3600,  # Allow jobs to run up to 1 hour late
+        'coalesce': True,            # If multiple runs were missed, only run once
+    }
+)
+
+
+def get_config_value(key: str, default: str = None) -> str:
+    """Get a config value from system_config"""
+    db = SyncSessionLocal()
+    try:
+        result = db.execute(
+            text("SELECT config_value FROM system_config WHERE config_key = :key"),
+            {"key": key}
+        )
+        row = result.fetchone()
+        if row and row.config_value:
+            return row.config_value
+        return default
+    except Exception as e:
+        logger.error(f"Error getting config {key}: {e}")
+        return default
+    finally:
+        db.close()
+
+
+def is_sync_enabled(source: str) -> bool:
+    """Check if a sync source is enabled in config"""
+    value = get_config_value(f"sync_{source}_enabled")
+    if value:
+        return value.lower() in ('true', '1', 'yes', 'enabled')
+    return False
+
+
+def get_sync_time(source: str, default_hour: int = 5, default_minute: int = 0) -> tuple:
+    """Get sync time for a source, returns (hour, minute) tuple"""
+    time_str = get_config_value(f"sync_{source}_time")
+    if time_str:
+        try:
+            parts = time_str.split(':')
+            return (int(parts[0]), int(parts[1]))
+        except (ValueError, IndexError):
+            logger.warning(f"Invalid time format for {source}: {time_str}, using default")
+    return (default_hour, default_minute)
+
+
+async def run_scheduled_newbook_sync():
+    """Wrapper to check if Newbook bookings sync is enabled before running"""
+    if is_sync_enabled("newbook_bookings"):
+        # Get sync type from config (incremental or full)
+        sync_type = get_config_value("sync_newbook_bookings_type", "incremental")
+        logger.info(f"Running scheduled Newbook bookings sync (mode={sync_type})")
+        # Run synchronous function in thread pool to avoid blocking
+        import asyncio
+        loop = asyncio.get_event_loop()
+        await loop.run_in_executor(
+            None,
+            run_bookings_data_sync,
+            sync_type,  # sync_mode
+            None,       # from_date
+            None,       # to_date
+            "scheduler" # triggered_by
+        )
+    else:
+        logger.debug("Scheduled Newbook bookings sync skipped (disabled in settings)")
+
+
+async def run_scheduled_resos_bookings_sync():
+    """Wrapper to check if Resos bookings sync is enabled before running"""
+    from datetime import date, timedelta
+    if is_sync_enabled("resos_bookings"):
+        logger.info("Running scheduled Resos bookings sync")
+        # Daily: -7 days to +365 days (recent history + forecast window)
+        from_date = date.today() - timedelta(days=7)
+        to_date = date.today() + timedelta(days=365)
+        await sync_resos_bookings_data(from_date, to_date, triggered_by="scheduler")
+    else:
+        logger.info("Scheduled Resos bookings sync skipped (disabled in settings)")
+
+
+async def run_scheduled_resos_sync():
+    """Wrapper to check if Resos sync is enabled before running"""
+    if is_sync_enabled("resos"):
+        logger.info("Running scheduled Resos sync")
+        await sync_resos_data(triggered_by="scheduler")
+    else:
+        logger.info("Scheduled Resos sync skipped (disabled in settings)")
+
+
+async def run_scheduled_occupancy_report_sync():
+    """Wrapper to run occupancy report sync (uses dedicated occupancy enabled flag)"""
+    from datetime import date, timedelta
+    if is_sync_enabled("newbook_occupancy"):
+        logger.info("Running scheduled Newbook occupancy report sync")
+        # Daily: -7 days to +365 days
+        from_date = date.today() - timedelta(days=7)
+        to_date = date.today() + timedelta(days=365)
+        await sync_newbook_occupancy_report(from_date, to_date, triggered_by="scheduler")
+    else:
+        logger.info("Scheduled Newbook occupancy report sync skipped (disabled in settings)")
+
+
+async def run_scheduled_earned_revenue_sync():
+    """Wrapper to run earned revenue sync (uses dedicated enabled flag)"""
+    from datetime import date, timedelta
+    if is_sync_enabled("newbook_earned_revenue"):
+        logger.info("Running scheduled Newbook earned revenue sync")
+        # Daily: last 7 days only (historical data, catches adjustments)
+        from_date = date.today() - timedelta(days=7)
+        to_date = date.today()
+        await sync_newbook_earned_revenue(from_date, to_date, triggered_by="scheduler")
+    else:
+        logger.info("Scheduled Newbook earned revenue sync skipped (disabled in settings)")
+
+
+async def run_scheduled_current_rates_sync():
+    """Wrapper to run current rates sync (for pickup-v2 upper bounds)"""
+    if is_sync_enabled("newbook_current_rates"):
+        logger.info("Running scheduled Newbook current rates sync")
+        await run_fetch_current_rates()
+    else:
+        logger.debug("Scheduled Newbook current rates sync skipped (disabled in settings)")
+
+
+def reschedule_sync_jobs():
+    """
+    Read sync times from config and reschedule sync jobs.
+    Called at startup and daily at 1am to pick up config changes.
+    """
+    logger.info("Rescheduling sync jobs from config...")
+
+    # Newbook bookings sync
+    nb_hour, nb_min = get_sync_time("newbook_bookings", 5, 0)
+    scheduler.add_job(
+        run_scheduled_newbook_sync,
+        CronTrigger(hour=nb_hour, minute=nb_min),
+        id="newbook_sync",
+        name=f"Daily Newbook Bookings Sync ({nb_hour:02d}:{nb_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Newbook bookings sync scheduled for {nb_hour:02d}:{nb_min:02d}")
+
+    # Resos bookings sync
+    rsb_hour, rsb_min = get_sync_time("resos_bookings", 5, 5)
+    scheduler.add_job(
+        run_scheduled_resos_bookings_sync,
+        CronTrigger(hour=rsb_hour, minute=rsb_min),
+        id="resos_bookings_sync",
+        name=f"Daily Resos Bookings Sync ({rsb_hour:02d}:{rsb_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Resos bookings sync scheduled for {rsb_hour:02d}:{rsb_min:02d}")
+
+    # Resos sync - uses general sync_schedule_time for now
+    rs_time = get_config_value("sync_schedule_time", "05:05")
+    try:
+        rs_hour, rs_min = int(rs_time.split(':')[0]), int(rs_time.split(':')[1])
+    except:
+        rs_hour, rs_min = 5, 5
+    scheduler.add_job(
+        run_scheduled_resos_sync,
+        CronTrigger(hour=rs_hour, minute=rs_min),
+        id="resos_sync",
+        name=f"Daily Resos Sync ({rs_hour:02d}:{rs_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Resos sync scheduled for {rs_hour:02d}:{rs_min:02d}")
+
+    # Newbook occupancy report
+    occ_hour, occ_min = get_sync_time("newbook_occupancy", 5, 8)
+    scheduler.add_job(
+        run_scheduled_occupancy_report_sync,
+        CronTrigger(hour=occ_hour, minute=occ_min),
+        id="newbook_occupancy_report",
+        name=f"Daily Newbook Occupancy Report ({occ_hour:02d}:{occ_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Newbook occupancy report scheduled for {occ_hour:02d}:{occ_min:02d}")
+
+    # Newbook earned revenue
+    rev_hour, rev_min = get_sync_time("newbook_earned_revenue", 5, 10)
+    scheduler.add_job(
+        run_scheduled_earned_revenue_sync,
+        CronTrigger(hour=rev_hour, minute=rev_min),
+        id="newbook_earned_revenue",
+        name=f"Daily Newbook Earned Revenue ({rev_hour:02d}:{rev_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Newbook earned revenue scheduled for {rev_hour:02d}:{rev_min:02d}")
+
+    # Aggregation - 15 mins after the latest sync job
+    latest_sync = max(nb_hour * 60 + nb_min, occ_hour * 60 + occ_min, rev_hour * 60 + rev_min)
+    agg_mins = latest_sync + 15
+    agg_hour, agg_min = agg_mins // 60, agg_mins % 60
+    scheduler.add_job(
+        run_aggregation,
+        CronTrigger(hour=agg_hour, minute=agg_min),
+        id="aggregation",
+        name=f"Daily Aggregation ({agg_hour:02d}:{agg_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Aggregation scheduled for {agg_hour:02d}:{agg_min:02d}")
+
+    logger.info("Sync jobs rescheduled successfully")
+
+
+def start_scheduler():
+    """Initialize and start the scheduler"""
+    logger.info("Starting scheduler...")
+
+    # 1am daily: reschedule sync jobs from config
+    # This picks up any config changes made during the day
+    scheduler.add_job(
+        reschedule_sync_jobs,
+        CronTrigger(hour=1, minute=0),
+        id="reschedule_sync_jobs",
+        name="Daily Reschedule Sync Jobs (01:00)",
+        replace_existing=True
+    )
+
+    # Schedule sync jobs from config (initial schedule)
+    reschedule_sync_jobs()
+
+    # Pickup snapshot - Daily at 5:30 AM
+    scheduler.add_job(
+        run_pickup_snapshot,
+        CronTrigger(hour=5, minute=30),
+        id="pickup_snapshot",
+        name="Daily Pickup Snapshot",
+        replace_existing=True
+    )
+
+    # Pace snapshot v2 - Daily at 5:32 AM
+    # Captures revenue pace for pickup-v2 model
+    scheduler.add_job(
+        run_pace_snapshot_v2,
+        CronTrigger(hour=5, minute=32),
+        id="pace_snapshot_v2",
+        name="Daily Pace Snapshot V2",
+        replace_existing=True
+    )
+
+    # Fetch current rates from Newbook - Daily at 5:20 AM
+    # Populates newbook_current_rates for pickup-v2 upper bound calculations
+    scheduler.add_job(
+        run_scheduled_current_rates_sync,
+        CronTrigger(hour=5, minute=20),
+        id="fetch_current_rates",
+        name="Daily Fetch Current Rates",
+        replace_existing=True
+    )
+
+    # Booking.com rate scraper - Daily at configured time (default 05:30)
+    # Tiered: daily 30d, weekly 31-180d (Mon-Fri), biweekly 181-365d (Wed)
+    booking_time = get_config_value("booking_scraper_daily_time", "05:30")
+    try:
+        bk_hour, bk_min = int(booking_time.split(':')[0]), int(booking_time.split(':')[1])
+    except (ValueError, IndexError):
+        bk_hour, bk_min = 5, 30
+    scheduler.add_job(
+        run_scheduled_booking_scrape_async,
+        CronTrigger(hour=bk_hour, minute=bk_min),
+        id="booking_scrape",
+        name=f"Daily Booking.com Scrape ({bk_hour:02d}:{bk_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  Booking.com scrape scheduled for {bk_hour:02d}:{bk_min:02d}")
+
+    # Daily forecast (0-28 days) - Daily at 6:00 AM
+    # Prophet, XGBoost, Pickup, and CatBoost for operational planning window
+    scheduler.add_job(
+        lambda: run_daily_forecast(horizon_days=28, models=['prophet', 'xgboost', 'pickup', 'catboost']),
+        CronTrigger(hour=6, minute=0),
+        id="forecast_daily",
+        name="Daily Forecast (0-28 days)",
+        replace_existing=True
+    )
+
+    # Long-term forecast (29-365 days) - Weekly on Monday at 6:30 AM
+    # Prophet and XGBoost only (pickup less useful at long range)
+    scheduler.add_job(
+        lambda: run_daily_forecast(horizon_days=365, start_days=29, models=['prophet', 'xgboost']),
+        CronTrigger(day_of_week="mon", hour=6, minute=30),
+        id="forecast_weekly",
+        name="Weekly Forecast (29-365 days)",
+        replace_existing=True
+    )
+
+    # Accuracy calculation - Daily at 7:00 AM
+    scheduler.add_job(
+        run_accuracy_calculation,
+        CronTrigger(hour=7, minute=0),
+        id="accuracy_calc",
+        name="Daily Accuracy Calculation",
+        replace_existing=True
+    )
+
+    # Weekly forecast snapshot - Monday at 6:00 AM (default)
+    # Get time from config (default: Monday 6:00 AM)
+    snapshot_time = get_config_value("forecast_snapshot_time", "06:00")
+    try:
+        snapshot_hour, snapshot_min = int(snapshot_time.split(':')[0]), int(snapshot_time.split(':')[1])
+    except:
+        snapshot_hour, snapshot_min = 6, 0
+
+    scheduler.add_job(
+        run_weekly_forecast_snapshot,
+        CronTrigger(day_of_week="mon", hour=snapshot_hour, minute=snapshot_min),
+        id="weekly_forecast_snapshot",
+        name=f"Weekly Forecast Snapshot (Mon {snapshot_hour:02d}:{snapshot_min:02d})",
+        replace_existing=True
+    )
+
+    # AI Insights - Daily at configurable time (default 07:15, after accuracy calc)
+    ai_time = get_config_value("ai_insights_schedule_time", "07:15")
+    try:
+        ai_hour, ai_min = int(ai_time.split(':')[0]), int(ai_time.split(':')[1])
+    except (ValueError, IndexError):
+        ai_hour, ai_min = 7, 15
+    scheduler.add_job(
+        run_ai_insights_generation,
+        CronTrigger(hour=ai_hour, minute=ai_min),
+        id="ai_insights",
+        name=f"Daily AI Insights ({ai_hour:02d}:{ai_min:02d})",
+        replace_existing=True
+    )
+    logger.info(f"  AI Insights scheduled for {ai_hour:02d}:{ai_min:02d}")
+
+    scheduler.start()
+    logger.info("Scheduler started successfully")
+
+
+def shutdown_scheduler():
+    """Shutdown the scheduler"""
+    logger.info("Shutting down scheduler...")
+    scheduler.shutdown()

backend/schema.sql

@@ -0,0 +1,870 @@
+-- Forecasting Application Database Schema
+-- PostgreSQL 15+
+-- Database: forecast_data (created via POSTGRES_DB env var)
+
+
+-- ============================================
+-- USERS & AUTHENTICATION
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS users (
+    id SERIAL PRIMARY KEY,
+    username VARCHAR(50) UNIQUE NOT NULL,
+    password_hash VARCHAR(255) NOT NULL,
+    display_name VARCHAR(100),
+    is_active BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+-- Default admin user (password: admin123 - change in production!)
+INSERT INTO users (username, password_hash, display_name) VALUES
+('admin', '$2b$12$LQv3c1yqBWVHxkd0LHAkCOYz6TtxMQJqhN8/X4.VTtYH0lHXG0Kku', 'Administrator')
+ON CONFLICT (username) DO NOTHING;
+
+-- ============================================
+-- API KEYS (for external integrations)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS api_keys (
+    id SERIAL PRIMARY KEY,
+    key_hash VARCHAR(64) NOT NULL UNIQUE,     -- SHA256 hash of key (never store plaintext)
+    key_prefix VARCHAR(20) NOT NULL,          -- First chars for display (e.g., "fk_abc123...")
+    name VARCHAR(100) NOT NULL,               -- Descriptive name (e.g., "Kitchen Flash App")
+    is_active BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMP DEFAULT NOW(),
+    last_used_at TIMESTAMP,
+    created_by VARCHAR(100)
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_keys_hash ON api_keys(key_hash);
+CREATE INDEX IF NOT EXISTS idx_api_keys_active ON api_keys(is_active);
+
+-- ============================================
+-- SYSTEM CONFIGURATION
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS system_config (
+    id SERIAL PRIMARY KEY,
+    config_key VARCHAR(100) NOT NULL UNIQUE,
+    config_value TEXT,
+    is_encrypted BOOLEAN DEFAULT FALSE,
+    description TEXT,
+    updated_at TIMESTAMP DEFAULT NOW(),
+    updated_by VARCHAR(100)
+);
+
+-- Default config entries
+INSERT INTO system_config (config_key, description) VALUES
+('newbook_api_key', 'Newbook API Key'),
+('newbook_username', 'Newbook Username'),
+('newbook_password', 'Newbook Password'),
+('newbook_region', 'Newbook Region Code'),
+('resos_api_key', 'Resos API Key'),
+('total_rooms', 'Total number of hotel rooms'),
+('hotel_name', 'Hotel/Property Name'),
+('timezone', 'Local timezone (e.g., Europe/London)'),
+('accommodation_vat_rate', 'VAT rate for accommodation (e.g., 0.20 for 20%)'),
+('sync_newbook_enabled', 'Enable automatic Newbook sync (true/false)'),
+('sync_resos_enabled', 'Enable automatic Resos sync (true/false)'),
+('sync_schedule_time', 'Time for daily sync (HH:MM format)'),
+('sync_newbook_bookings_enabled', 'Enable automatic Newbook bookings data sync (true/false)'),
+('sync_newbook_bookings_type', 'Newbook bookings sync type (incremental/full)'),
+('sync_newbook_bookings_time', 'Newbook bookings sync time (HH:MM)'),
+('sync_newbook_occupancy_enabled', 'Enable automatic Newbook occupancy report sync (true/false)'),
+('sync_newbook_occupancy_time', 'Newbook occupancy report sync time (HH:MM)'),
+('last_bookings_aggregation_at', 'Timestamp of last bookings stats aggregation'),
+('sync_newbook_earned_revenue_enabled', 'Enable automatic Newbook earned revenue sync (true/false)'),
+('sync_newbook_earned_revenue_time', 'Newbook earned revenue sync time (HH:MM)'),
+('last_revenue_aggregation_at', 'Timestamp of last revenue aggregation'),
+('sync_newbook_current_rates_enabled', 'Enable automatic Newbook current rates sync for pickup-v2 (true/false)'),
+('sync_newbook_current_rates_time', 'Newbook current rates sync time (HH:MM)')
+ON CONFLICT (config_key) DO NOTHING;
+
+-- Set defaults
+UPDATE system_config SET config_value = '80' WHERE config_key = 'total_rooms' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'Europe/London' WHERE config_key = 'timezone' AND config_value IS NULL;
+UPDATE system_config SET config_value = '0.20' WHERE config_key = 'accommodation_vat_rate' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_resos_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:00' WHERE config_key = 'sync_schedule_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_bookings_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'incremental' WHERE config_key = 'sync_newbook_bookings_type' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:00' WHERE config_key = 'sync_newbook_bookings_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_occupancy_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:00' WHERE config_key = 'sync_newbook_occupancy_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_earned_revenue_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:10' WHERE config_key = 'sync_newbook_earned_revenue_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_current_rates_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:20' WHERE config_key = 'sync_newbook_current_rates_time' AND config_value IS NULL;
+
+-- ============================================
+-- TAX RATES (date-based tax configuration)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS tax_rates (
+    id SERIAL PRIMARY KEY,
+    tax_type VARCHAR(50) NOT NULL,          -- 'accommodation_vat', 'food_vat', etc.
+    rate DECIMAL(5,4) NOT NULL,             -- e.g., 0.20 for 20%
+    effective_from DATE NOT NULL,           -- Date this rate becomes effective
+    created_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(tax_type, effective_from)
+);
+
+CREATE INDEX IF NOT EXISTS idx_tax_rates_type ON tax_rates(tax_type);
+CREATE INDEX IF NOT EXISTS idx_tax_rates_effective ON tax_rates(tax_type, effective_from);
+
+-- Default accommodation VAT rate (20% from 2022-01-01)
+INSERT INTO tax_rates (tax_type, rate, effective_from) VALUES
+('accommodation_vat', 0.20, '2022-01-01')
+ON CONFLICT (tax_type, effective_from) DO NOTHING;
+
+-- ============================================
+-- NEWBOOK ROOM CATEGORIES (for occupancy settings)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_room_categories (
+    id SERIAL PRIMARY KEY,
+    site_id VARCHAR(50) NOT NULL UNIQUE,
+    site_name VARCHAR(255) NOT NULL,
+    site_type VARCHAR(100),
+    room_count INTEGER DEFAULT 0,
+    is_included BOOLEAN DEFAULT TRUE,
+    display_order INTEGER DEFAULT 0,
+    fetched_at TIMESTAMP DEFAULT NOW()
+);
+
+-- ============================================
+-- NEWBOOK GL ACCOUNTS (for revenue mapping)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_gl_accounts (
+    id SERIAL PRIMARY KEY,
+    gl_account_id VARCHAR(50) NOT NULL UNIQUE,
+    gl_code VARCHAR(50),
+    gl_name VARCHAR(255),
+    gl_group_id VARCHAR(50),
+    gl_group_name VARCHAR(255),
+    department VARCHAR(20),  -- 'accommodation', 'dry', 'wet', or null
+    last_seen_date DATE,
+    total_amount DECIMAL(14,2) DEFAULT 0,
+    is_active BOOLEAN DEFAULT TRUE,
+    fetched_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_gl_accounts_department ON newbook_gl_accounts(department);
+CREATE INDEX IF NOT EXISTS idx_gl_accounts_group ON newbook_gl_accounts(gl_group_name);
+
+-- ============================================
+-- NEWBOOK BOOKINGS DATA (historical booking data)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_bookings_data (
+    id SERIAL PRIMARY KEY,
+    newbook_id VARCHAR(50) NOT NULL UNIQUE,
+    booking_reference VARCHAR(100),
+    bookings_group_id VARCHAR(50),
+    booking_placed TIMESTAMP,  -- When booking was created (for lead time calculations)
+    arrival_date DATE NOT NULL,
+    departure_date DATE NOT NULL,
+    nights INTEGER,
+    adults INTEGER DEFAULT 0,
+    children INTEGER DEFAULT 0,
+    infants INTEGER DEFAULT 0,
+    total_guests INTEGER,
+    category_id VARCHAR(50),
+    room_type VARCHAR(100),
+    site_id VARCHAR(50),
+    room_number VARCHAR(50),
+    status VARCHAR(50),
+    total_amount DECIMAL(12,2),
+    tariff_name VARCHAR(255),
+    tariff_total DECIMAL(12,2),
+    travel_agent_id VARCHAR(50),
+    travel_agent_name VARCHAR(255),
+    travel_agent_commission DECIMAL(12,2),
+    booking_source_id VARCHAR(50),
+    booking_source_name VARCHAR(255),
+    booking_parent_source_id VARCHAR(50),
+    booking_parent_source_name VARCHAR(255),
+    booking_method_id VARCHAR(50),
+    booking_method_name VARCHAR(100),
+    raw_json JSONB,
+    fetched_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_bookings_arrival ON newbook_bookings_data(arrival_date);
+CREATE INDEX IF NOT EXISTS idx_bookings_status ON newbook_bookings_data(status);
+CREATE INDEX IF NOT EXISTS idx_bookings_placed ON newbook_bookings_data(booking_placed);
+
+-- ============================================
+-- NEWBOOK EARNED REVENUE DATA (historical revenue data)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_earned_revenue_data (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    gl_account_id VARCHAR(50),
+    gl_code VARCHAR(50),
+    gl_name VARCHAR(255),
+    amount_gross DECIMAL(12,2) DEFAULT 0,
+    amount_net DECIMAL(12,2) DEFAULT 0,
+    revenue_type VARCHAR(30),
+    fetched_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, gl_account_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_earned_revenue_data_date ON newbook_earned_revenue_data(date);
+CREATE INDEX IF NOT EXISTS idx_earned_revenue_data_type ON newbook_earned_revenue_data(date, revenue_type);
+
+-- ============================================
+-- NEWBOOK NET REVENUE DATA (aggregated by department)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_net_revenue_data (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL UNIQUE,
+    accommodation DECIMAL(12,2) DEFAULT 0,  -- Net accommodation revenue
+    dry DECIMAL(12,2) DEFAULT 0,            -- Net dry (food) revenue
+    wet DECIMAL(12,2) DEFAULT 0,            -- Net wet (beverage) revenue
+    aggregated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_net_revenue_data_date ON newbook_net_revenue_data(date);
+
+-- ============================================
+-- NEWBOOK OCCUPANCY REPORT DATA (official capacity & occupancy)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_occupancy_report_data (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    category_id VARCHAR(50) NOT NULL,
+    category_name VARCHAR(255),
+    available INTEGER DEFAULT 0,       -- Total configured rooms for category
+    occupied INTEGER DEFAULT 0,        -- Official occupied per Newbook
+    maintenance INTEGER DEFAULT 0,     -- Rooms offline (deduct from available for bookable rooms)
+    allotted INTEGER DEFAULT 0,        -- Block allocations
+    revenue_gross DECIMAL(12,2) DEFAULT 0,
+    revenue_net DECIMAL(12,2) DEFAULT 0,
+    occupancy_pct DECIMAL(5,2),
+    fetched_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, category_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_occupancy_report_data_date ON newbook_occupancy_report_data(date);
+
+-- ============================================
+-- SYNC LOGGING
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS sync_log (
+    id SERIAL PRIMARY KEY,
+    sync_type VARCHAR(30) NOT NULL,
+    source VARCHAR(20) NOT NULL,
+    started_at TIMESTAMP NOT NULL,
+    completed_at TIMESTAMP,
+    status VARCHAR(20) NOT NULL,
+    records_fetched INTEGER,
+    records_created INTEGER,
+    records_updated INTEGER,
+    date_from DATE,
+    date_to DATE,
+    error_message TEXT,
+    triggered_by VARCHAR(100)
+);
+
+-- ============================================
+-- NEWBOOK BOOKINGS STATS (aggregated daily stats)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_bookings_stats (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL UNIQUE,
+
+    -- Room availability (from newbook_occupancy_report_data)
+    rooms_count INTEGER DEFAULT 0,           -- Total available rooms (included categories)
+    maintenance_count INTEGER DEFAULT 0,     -- Rooms offline/maintenance
+    bookable_count INTEGER DEFAULT 0,        -- rooms_count - maintenance_count
+
+    -- Occupancy totals (from bookings)
+    booking_count INTEGER DEFAULT 0,         -- Occupied rooms (bookings staying this night)
+    guests_count INTEGER DEFAULT 0,
+    adults_count INTEGER DEFAULT 0,
+    children_count INTEGER DEFAULT 0,
+    infants_count INTEGER DEFAULT 0,
+
+    -- Occupancy percentages
+    total_occupancy_pct DECIMAL(5,2),        -- booking_count / rooms_count * 100
+    bookable_occupancy_pct DECIMAL(5,2),     -- booking_count / bookable_count * 100
+
+    -- Revenue totals
+    guest_rate_total DECIMAL(12,2) DEFAULT 0,      -- SUM of calculated_amount (gross)
+    net_booking_rev_total DECIMAL(12,2) DEFAULT 0, -- SUM of net accommodation
+
+    -- Per-category breakdowns (JSONB)
+    occupancy_by_category JSONB DEFAULT '{}',
+    revenue_by_category JSONB DEFAULT '{}',
+    availability_by_category JSONB DEFAULT '{}',
+
+    -- Pickup-V2: Rate statistics per category for bounds calculation
+    -- Structure: { "category_id": { "min_net": 120, "max_net": 200, "adr_net": 155, "rooms": 12 } }
+    rate_stats_by_category JSONB DEFAULT '{}',
+
+    aggregated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_bookings_stats_date ON newbook_bookings_stats(date);
+
+-- ============================================
+-- NEWBOOK BOOKING PACE (lead-time snapshots for forecasting)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_booking_pace (
+    id SERIAL PRIMARY KEY,
+    arrival_date DATE NOT NULL UNIQUE,
+
+    -- Monthly intervals: months 7-12 (6 columns)
+    d365 INTEGER,    -- 12 months out
+    d330 INTEGER,    -- 11 months out
+    d300 INTEGER,    -- 10 months out
+    d270 INTEGER,    -- 9 months out
+    d240 INTEGER,    -- 8 months out
+    d210 INTEGER,    -- 7 months out
+
+    -- Weekly intervals: weeks 5-25 (21 columns)
+    d177 INTEGER, d170 INTEGER, d163 INTEGER, d156 INTEGER, d149 INTEGER,
+    d142 INTEGER, d135 INTEGER, d128 INTEGER, d121 INTEGER, d114 INTEGER,
+    d107 INTEGER, d100 INTEGER, d93 INTEGER, d86 INTEGER, d79 INTEGER,
+    d72 INTEGER, d65 INTEGER, d58 INTEGER, d51 INTEGER, d44 INTEGER, d37 INTEGER,
+
+    -- Daily intervals: days 0-30 (31 columns)
+    d30 INTEGER, d29 INTEGER, d28 INTEGER, d27 INTEGER, d26 INTEGER,
+    d25 INTEGER, d24 INTEGER, d23 INTEGER, d22 INTEGER, d21 INTEGER,
+    d20 INTEGER, d19 INTEGER, d18 INTEGER, d17 INTEGER, d16 INTEGER,
+    d15 INTEGER, d14 INTEGER, d13 INTEGER, d12 INTEGER, d11 INTEGER,
+    d10 INTEGER, d9 INTEGER, d8 INTEGER, d7 INTEGER, d6 INTEGER,
+    d5 INTEGER, d4 INTEGER, d3 INTEGER, d2 INTEGER, d1 INTEGER, d0 INTEGER,
+
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_booking_pace_arrival ON newbook_booking_pace(arrival_date);
+
+-- ============================================
+-- PICKUP-V2: CATEGORY BOOKING PACE (per-category room counts at lead times)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS category_booking_pace (
+    id SERIAL PRIMARY KEY,
+    arrival_date DATE NOT NULL,
+    category_id VARCHAR(50) NOT NULL,
+
+    -- Monthly intervals: months 7-12 (6 columns)
+    d365 INTEGER, d330 INTEGER, d300 INTEGER, d270 INTEGER, d240 INTEGER, d210 INTEGER,
+
+    -- Weekly intervals: weeks 5-25 (21 columns)
+    d177 INTEGER, d170 INTEGER, d163 INTEGER, d156 INTEGER, d149 INTEGER,
+    d142 INTEGER, d135 INTEGER, d128 INTEGER, d121 INTEGER, d114 INTEGER,
+    d107 INTEGER, d100 INTEGER, d93 INTEGER, d86 INTEGER, d79 INTEGER,
+    d72 INTEGER, d65 INTEGER, d58 INTEGER, d51 INTEGER, d44 INTEGER, d37 INTEGER,
+
+    -- Daily intervals: days 0-30 (31 columns)
+    d30 INTEGER, d29 INTEGER, d28 INTEGER, d27 INTEGER, d26 INTEGER,
+    d25 INTEGER, d24 INTEGER, d23 INTEGER, d22 INTEGER, d21 INTEGER,
+    d20 INTEGER, d19 INTEGER, d18 INTEGER, d17 INTEGER, d16 INTEGER,
+    d15 INTEGER, d14 INTEGER, d13 INTEGER, d12 INTEGER, d11 INTEGER,
+    d10 INTEGER, d9 INTEGER, d8 INTEGER, d7 INTEGER, d6 INTEGER,
+    d5 INTEGER, d4 INTEGER, d3 INTEGER, d2 INTEGER, d1 INTEGER, d0 INTEGER,
+
+    updated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(arrival_date, category_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_category_booking_pace_arrival ON category_booking_pace(arrival_date);
+CREATE INDEX IF NOT EXISTS idx_category_booking_pace_category ON category_booking_pace(category_id);
+
+-- ============================================
+-- PICKUP-V2: REVENUE PACE (booked accommodation revenue at lead times)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS revenue_pace (
+    id SERIAL PRIMARY KEY,
+    stay_date DATE NOT NULL UNIQUE,
+
+    -- Monthly intervals: months 7-12 (6 columns) - DECIMAL for revenue
+    d365 DECIMAL(12,2), d330 DECIMAL(12,2), d300 DECIMAL(12,2),
+    d270 DECIMAL(12,2), d240 DECIMAL(12,2), d210 DECIMAL(12,2),
+
+    -- Weekly intervals: weeks 5-25 (21 columns)
+    d177 DECIMAL(12,2), d170 DECIMAL(12,2), d163 DECIMAL(12,2), d156 DECIMAL(12,2), d149 DECIMAL(12,2),
+    d142 DECIMAL(12,2), d135 DECIMAL(12,2), d128 DECIMAL(12,2), d121 DECIMAL(12,2), d114 DECIMAL(12,2),
+    d107 DECIMAL(12,2), d100 DECIMAL(12,2), d93 DECIMAL(12,2), d86 DECIMAL(12,2), d79 DECIMAL(12,2),
+    d72 DECIMAL(12,2), d65 DECIMAL(12,2), d58 DECIMAL(12,2), d51 DECIMAL(12,2), d44 DECIMAL(12,2), d37 DECIMAL(12,2),
+
+    -- Daily intervals: days 0-30 (31 columns)
+    d30 DECIMAL(12,2), d29 DECIMAL(12,2), d28 DECIMAL(12,2), d27 DECIMAL(12,2), d26 DECIMAL(12,2),
+    d25 DECIMAL(12,2), d24 DECIMAL(12,2), d23 DECIMAL(12,2), d22 DECIMAL(12,2), d21 DECIMAL(12,2),
+    d20 DECIMAL(12,2), d19 DECIMAL(12,2), d18 DECIMAL(12,2), d17 DECIMAL(12,2), d16 DECIMAL(12,2),
+    d15 DECIMAL(12,2), d14 DECIMAL(12,2), d13 DECIMAL(12,2), d12 DECIMAL(12,2), d11 DECIMAL(12,2),
+    d10 DECIMAL(12,2), d9 DECIMAL(12,2), d8 DECIMAL(12,2), d7 DECIMAL(12,2), d6 DECIMAL(12,2),
+    d5 DECIMAL(12,2), d4 DECIMAL(12,2), d3 DECIMAL(12,2), d2 DECIMAL(12,2), d1 DECIMAL(12,2), d0 DECIMAL(12,2),
+
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_revenue_pace_stay ON revenue_pace(stay_date);
+
+-- ============================================
+-- PICKUP-V2: CURRENT RATES FROM NEWBOOK (for ceiling calculations)
+-- Now with rate history tracking - stores snapshots when rates change
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_current_rates (
+    id SERIAL PRIMARY KEY,
+    category_id VARCHAR(50) NOT NULL,
+    rate_date DATE NOT NULL,
+    rate_name VARCHAR(255),
+    rate_gross DECIMAL(12,2),
+    rate_net DECIMAL(12,2),
+    tariffs_data JSONB DEFAULT '{}',  -- All available tariff options with availability status
+    valid_from TIMESTAMP DEFAULT NOW(),  -- When this rate version started
+    last_verified_at TIMESTAMP DEFAULT NOW()  -- Last time we confirmed rate is still current
+    -- No UNIQUE constraint - allows multiple versions per (category_id, rate_date)
+);
+
+CREATE INDEX IF NOT EXISTS idx_current_rates_date ON newbook_current_rates(rate_date);
+CREATE INDEX IF NOT EXISTS idx_current_rates_category ON newbook_current_rates(category_id);
+CREATE INDEX IF NOT EXISTS idx_current_rates_tariffs ON newbook_current_rates USING gin(tariffs_data);
+CREATE INDEX IF NOT EXISTS idx_current_rates_latest ON newbook_current_rates(category_id, rate_date, valid_from DESC);
+
+-- Migration: Add tariffs_data column if missing (for existing databases)
+ALTER TABLE newbook_current_rates ADD COLUMN IF NOT EXISTS tariffs_data JSONB DEFAULT '{}';
+
+-- Migration: Convert from old schema to new snapshot schema
+-- Rename fetched_at to valid_from if it exists
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns
+               WHERE table_name = 'newbook_current_rates' AND column_name = 'fetched_at') THEN
+        ALTER TABLE newbook_current_rates RENAME COLUMN fetched_at TO valid_from;
+    END IF;
+END $$;
+
+-- Add last_verified_at column if missing
+ALTER TABLE newbook_current_rates ADD COLUMN IF NOT EXISTS last_verified_at TIMESTAMP DEFAULT NOW();
+
+-- Drop unique constraint if it exists (allows rate history)
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM pg_constraint
+               WHERE conname = 'newbook_current_rates_category_id_rate_date_key') THEN
+        ALTER TABLE newbook_current_rates
+        DROP CONSTRAINT newbook_current_rates_category_id_rate_date_key;
+    END IF;
+END $$;
+
+-- ============================================
+-- FORECASTING TABLES (for Prophet, XGBoost, CatBoost models)
+-- ============================================
+
+-- Forecast metrics configuration
+CREATE TABLE IF NOT EXISTS forecast_metrics (
+    id SERIAL PRIMARY KEY,
+    metric_code VARCHAR(50) NOT NULL UNIQUE,
+    metric_name VARCHAR(100) NOT NULL,
+    description TEXT,
+    unit VARCHAR(20),
+    is_active BOOLEAN DEFAULT TRUE,
+    use_prophet BOOLEAN DEFAULT TRUE,
+    use_xgboost BOOLEAN DEFAULT TRUE,
+    use_pickup BOOLEAN DEFAULT FALSE,
+    use_catboost BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+-- Default metrics for forecasting
+INSERT INTO forecast_metrics (metric_code, metric_name, description, unit, use_prophet, use_xgboost, use_pickup, use_catboost) VALUES
+('hotel_occupancy_pct', 'Hotel Occupancy %', 'Percentage of available rooms occupied', '%', TRUE, TRUE, TRUE, TRUE),
+('hotel_room_nights', 'Room Nights', 'Number of rooms sold', 'rooms', TRUE, TRUE, TRUE, TRUE),
+('hotel_guests', 'Guest Count', 'Total guests staying', 'guests', TRUE, TRUE, FALSE, TRUE),
+('hotel_arrivals', 'Arrivals', 'Number of check-ins', 'arrivals', TRUE, TRUE, FALSE, TRUE)
+ON CONFLICT (metric_code) DO NOTHING;
+
+-- Daily metrics (actuals storage - populated from newbook_bookings_stats)
+CREATE TABLE IF NOT EXISTS daily_metrics (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    metric_code VARCHAR(50) NOT NULL,
+    actual_value DECIMAL(12,2),
+    source VARCHAR(50) DEFAULT 'newbook',
+    updated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, metric_code)
+);
+
+CREATE INDEX IF NOT EXISTS idx_daily_metrics_date ON daily_metrics(date);
+CREATE INDEX IF NOT EXISTS idx_daily_metrics_code ON daily_metrics(metric_code, date);
+
+-- Forecasts storage
+CREATE TABLE IF NOT EXISTS forecasts (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    model_type VARCHAR(20) NOT NULL,  -- 'prophet', 'xgboost', 'pickup', 'catboost'
+    predicted_value DECIMAL(12,2) NOT NULL,
+    lower_bound DECIMAL(12,2),
+    upper_bound DECIMAL(12,2),
+    generated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(forecast_date, forecast_type, model_type, generated_at)
+);
+
+CREATE INDEX IF NOT EXISTS idx_forecasts_date ON forecasts(forecast_date);
+CREATE INDEX IF NOT EXISTS idx_forecasts_type ON forecasts(forecast_type, model_type);
+CREATE INDEX IF NOT EXISTS idx_forecasts_generated ON forecasts(generated_at DESC);
+
+-- Actual vs forecast comparison
+CREATE TABLE IF NOT EXISTS actual_vs_forecast (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    metric_type VARCHAR(50) NOT NULL,
+    actual_value DECIMAL(12,2),
+    budget_value DECIMAL(12,2),
+    -- Prophet
+    prophet_forecast DECIMAL(12,2),
+    prophet_lower DECIMAL(12,2),
+    prophet_upper DECIMAL(12,2),
+    prophet_error DECIMAL(12,4),
+    prophet_pct_error DECIMAL(8,4),
+    -- XGBoost
+    xgboost_forecast DECIMAL(12,2),
+    xgboost_error DECIMAL(12,4),
+    xgboost_pct_error DECIMAL(8,4),
+    -- Pickup
+    pickup_forecast DECIMAL(12,2),
+    pickup_error DECIMAL(12,4),
+    pickup_pct_error DECIMAL(8,4),
+    -- CatBoost
+    catboost_forecast DECIMAL(12,2),
+    catboost_error DECIMAL(12,4),
+    catboost_pct_error DECIMAL(8,4),
+    -- Analysis
+    best_model VARCHAR(20),
+    calculated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, metric_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_actual_vs_forecast_date ON actual_vs_forecast(date);
+CREATE INDEX IF NOT EXISTS idx_actual_vs_forecast_type ON actual_vs_forecast(metric_type);
+
+-- Prophet decomposition storage
+CREATE TABLE IF NOT EXISTS prophet_decomposition (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    trend DECIMAL(12,2),
+    yearly_seasonality DECIMAL(12,2),
+    weekly_seasonality DECIMAL(12,2),
+    daily_seasonality DECIMAL(12,2),
+    holiday_effects JSONB,
+    regressor_effects JSONB,
+    generated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_prophet_decomposition_date ON prophet_decomposition(forecast_date, forecast_type);
+
+-- XGBoost SHAP explanations
+CREATE TABLE IF NOT EXISTS xgboost_explanations (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    base_value DECIMAL(12,2),
+    feature_values JSONB,
+    shap_values JSONB,
+    top_positive JSONB,
+    top_negative JSONB,
+    generated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_xgboost_explanations_date ON xgboost_explanations(forecast_date, forecast_type);
+
+-- Pickup model explanations
+CREATE TABLE IF NOT EXISTS pickup_explanations (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    current_otb DECIMAL(12,2),
+    days_out INTEGER,
+    comparison_date DATE,
+    comparison_otb DECIMAL(12,2),
+    comparison_final DECIMAL(12,2),
+    pickup_curve_pct DECIMAL(8,4),
+    pickup_curve_stddev DECIMAL(8,4),
+    pace_vs_prior_pct DECIMAL(8,4),
+    projection_method VARCHAR(50),
+    projected_value DECIMAL(12,2),
+    confidence_note TEXT,
+    generated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_pickup_explanations_date ON pickup_explanations(forecast_date, forecast_type);
+
+-- Monthly budgets from FD
+CREATE TABLE IF NOT EXISTS monthly_budgets (
+    id SERIAL PRIMARY KEY,
+    year INTEGER NOT NULL,
+    month INTEGER NOT NULL CHECK (month >= 1 AND month <= 12),
+    budget_type VARCHAR(50) NOT NULL,
+    budget_value DECIMAL(12,2) NOT NULL,
+    notes TEXT,
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(year, month, budget_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_monthly_budgets_year ON monthly_budgets(year, budget_type);
+
+-- Daily budgets (distributed from monthly)
+CREATE TABLE IF NOT EXISTS daily_budgets (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    budget_type VARCHAR(50) NOT NULL,
+    budget_value DECIMAL(12,2),
+    distribution_method VARCHAR(50),
+    prior_year_pct DECIMAL(10,6),
+    monthly_budget_id INTEGER REFERENCES monthly_budgets(id),
+    calculated_at TIMESTAMP,
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP,
+    UNIQUE(date, budget_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_daily_budgets_date ON daily_budgets(date, budget_type);
+
+-- Forecast snapshots (for tracking how forecasts evolve over time)
+CREATE TABLE IF NOT EXISTS forecast_snapshots (
+    id SERIAL PRIMARY KEY,
+    snapshot_date DATE NOT NULL,
+    target_date DATE NOT NULL,
+    metric_code VARCHAR(50) NOT NULL,
+    days_out INTEGER NOT NULL,
+    prophet_value DECIMAL(12,2),
+    xgboost_value DECIMAL(12,2),
+    pickup_value DECIMAL(12,2),
+    catboost_value DECIMAL(12,2),
+    created_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(snapshot_date, target_date, metric_code)
+);
+
+CREATE INDEX IF NOT EXISTS idx_forecast_snapshots_target ON forecast_snapshots(target_date, metric_code);
+
+-- ============================================
+-- USER ROLES (migration for existing users table)
+-- ============================================
+
+ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR(20) DEFAULT 'admin';
+
+-- ============================================
+-- RECONCILIATION: CASH UP SESSIONS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_cash_ups (
+    id SERIAL PRIMARY KEY,
+    session_date DATE NOT NULL UNIQUE,
+    created_by INTEGER NOT NULL REFERENCES users(id),
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW(),
+    status VARCHAR(20) DEFAULT 'draft' CHECK (status IN ('draft', 'final')),
+    total_float_counted DECIMAL(10,2) DEFAULT 0.00,
+    total_cash_counted DECIMAL(10,2) DEFAULT 0.00,
+    notes TEXT,
+    submitted_at TIMESTAMP,
+    submitted_by INTEGER REFERENCES users(id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_cash_ups_date ON recon_cash_ups(session_date);
+CREATE INDEX IF NOT EXISTS idx_recon_cash_ups_status ON recon_cash_ups(status);
+
+-- ============================================
+-- RECONCILIATION: DENOMINATION COUNTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_denominations (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    count_type VARCHAR(20) NOT NULL DEFAULT 'takings' CHECK (count_type IN ('float', 'takings')),
+    denomination_type VARCHAR(20) NOT NULL CHECK (denomination_type IN ('note', 'coin')),
+    denomination_value DECIMAL(10,2) NOT NULL,
+    quantity INTEGER DEFAULT NULL,
+    value_entered DECIMAL(10,2) DEFAULT NULL,
+    total_amount DECIMAL(10,2) NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_denoms_cashup ON recon_denominations(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: CARD MACHINE DATA
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_card_machines (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    machine_name VARCHAR(100) NOT NULL,
+    total_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    amex_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    visa_mc_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_cards_cashup ON recon_card_machines(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: NEWBOOK PAYMENT RECORDS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_payment_records (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER REFERENCES recon_cash_ups(id) ON DELETE SET NULL,
+    newbook_payment_id VARCHAR(100),
+    booking_id VARCHAR(100),
+    guest_name VARCHAR(255),
+    payment_date TIMESTAMP NOT NULL,
+    payment_type VARCHAR(100),
+    payment_method VARCHAR(50),
+    transaction_method VARCHAR(50),
+    card_type VARCHAR(50),
+    amount DECIMAL(10,2) NOT NULL,
+    tendered DECIMAL(10,2),
+    processed_by VARCHAR(255),
+    item_type VARCHAR(50),
+    synced_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_payments_cashup ON recon_payment_records(cash_up_id);
+CREATE INDEX IF NOT EXISTS idx_recon_payments_date ON recon_payment_records(payment_date);
+
+-- ============================================
+-- RECONCILIATION: RECONCILIATION RESULTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_reconciliation (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    category VARCHAR(50) NOT NULL,
+    banked_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    reported_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    variance DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_recon_cashup ON recon_reconciliation(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: DAILY STATISTICS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_daily_stats (
+    id SERIAL PRIMARY KEY,
+    business_date DATE NOT NULL UNIQUE,
+    gross_sales DECIMAL(10,2) DEFAULT 0.00,
+    debtors_creditors_balance DECIMAL(10,2) DEFAULT 0.00,
+    rooms_sold INTEGER DEFAULT 0,
+    total_people INTEGER DEFAULT 0,
+    source VARCHAR(50) DEFAULT 'manual',
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_daily_date ON recon_daily_stats(business_date);
+
+-- ============================================
+-- RECONCILIATION: SALES BREAKDOWN
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_sales_breakdown (
+    id SERIAL PRIMARY KEY,
+    business_date DATE NOT NULL,
+    category VARCHAR(100) NOT NULL,
+    net_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_sales_date ON recon_sales_breakdown(business_date);
+
+-- ============================================
+-- RECONCILIATION: FLOAT COUNTS (Petty Cash, Change Tin, Safe Cash)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_float_counts (
+    id SERIAL PRIMARY KEY,
+    count_type VARCHAR(20) NOT NULL CHECK (count_type IN ('petty_cash', 'change_tin', 'safe_cash')),
+    count_date TIMESTAMP NOT NULL,
+    created_by INTEGER NOT NULL REFERENCES users(id),
+    created_at TIMESTAMP DEFAULT NOW(),
+    total_counted DECIMAL(10,2) DEFAULT 0.00,
+    total_receipts DECIMAL(10,2) DEFAULT 0.00,
+    target_amount DECIMAL(10,2) DEFAULT 0.00,
+    variance DECIMAL(10,2) DEFAULT 0.00,
+    notes TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_floats_type ON recon_float_counts(count_type);
+CREATE INDEX IF NOT EXISTS idx_recon_floats_date ON recon_float_counts(count_date);
+
+-- ============================================
+-- RECONCILIATION: FLOAT DENOMINATION COUNTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_float_denominations (
+    id SERIAL PRIMARY KEY,
+    float_count_id INTEGER NOT NULL REFERENCES recon_float_counts(id) ON DELETE CASCADE,
+    denomination_value DECIMAL(10,2) NOT NULL,
+    quantity INTEGER DEFAULT 0,
+    total_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_float_denoms_fc ON recon_float_denominations(float_count_id);
+
+-- ============================================
+-- RECONCILIATION: FLOAT RECEIPTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_float_receipts (
+    id SERIAL PRIMARY KEY,
+    float_count_id INTEGER NOT NULL REFERENCES recon_float_counts(id) ON DELETE CASCADE,
+    receipt_value DECIMAL(10,2) NOT NULL,
+    receipt_description VARCHAR(255)
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_float_receipts_fc ON recon_float_receipts(float_count_id);
+
+-- ============================================
+-- RECONCILIATION: CASH COUNT ATTACHMENTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_attachments (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    file_name VARCHAR(255) NOT NULL,
+    file_path VARCHAR(500) NOT NULL,
+    file_type VARCHAR(50) NOT NULL,
+    file_size BIGINT NOT NULL,
+    uploaded_by INTEGER NOT NULL REFERENCES users(id),
+    uploaded_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_attachments_cashup ON recon_attachments(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: SYSTEM CONFIG ENTRIES
+-- ============================================
+
+INSERT INTO system_config (config_key, config_value, description) VALUES
+('recon_expected_till_float', '300.00', 'Expected till float amount (GBP)'),
+('recon_variance_threshold', '10.00', 'Variance threshold for highlighting (GBP)'),
+('recon_default_report_days', '7', 'Default number of days for multi-day reports'),
+('recon_petty_cash_target', '200.00', 'Target amount for petty cash float'),
+('recon_change_tin_breakdown', '{"50.00":0,"20.00":0,"10.00":0,"5.00":0,"2.00":20.00,"1.00":20.00,"0.50":10.00,"0.20":10.00,"0.10":5.00,"0.05":5.00}', 'Change tin denomination breakdown targets (JSON)'),
+('recon_denominations', '{"notes":[50.00,20.00,10.00,5.00],"coins":[2.00,1.00,0.50,0.20,0.10,0.05,0.02,0.01]}', 'GBP denominations configuration (JSON)'),
+('recon_sales_breakdown_columns', '[]', 'Sales breakdown column configuration (JSON)'),
+('recon_safe_cash_target', '0.00', 'Target amount for safe cash float')
+ON CONFLICT (config_key) DO NOTHING;

backend/services/__init__.py

@@ -0,0 +1 @@
+# Services

backend/services/backup_service.py

@@ -0,0 +1,539 @@
+"""
+Backup and Restore Service
+
+Handles creating full backups of the database and files, and restoring from backups.
+"""
+import os
+import json
+import logging
+import shutil
+import subprocess
+import tempfile
+import zipfile
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Tuple, Dict, Any
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+# Backup storage directory
+BACKUP_DIR = Path("/app/data/backups")
+DATA_DIR = Path("/app/data")
+
+
+class BackupService:
+    """Service for managing backups and restores"""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+        self.ensure_backup_dir()
+
+    def ensure_backup_dir(self):
+        """Ensure backup directory exists"""
+        BACKUP_DIR.mkdir(parents=True, exist_ok=True)
+
+    async def ensure_backup_table_exists(self):
+        """Create backup_history table if it doesn't exist"""
+        await self.db.execute(text("""
+            CREATE TABLE IF NOT EXISTS backup_history (
+                id SERIAL PRIMARY KEY,
+                backup_type VARCHAR(20) NOT NULL,
+                status VARCHAR(20) NOT NULL,
+                filename VARCHAR(255),
+                file_path TEXT,
+                file_size_bytes BIGINT,
+                snapshot_count INTEGER,
+                file_count INTEGER,
+                started_at TIMESTAMP DEFAULT NOW(),
+                completed_at TIMESTAMP,
+                error_message TEXT,
+                created_by VARCHAR(100)
+            )
+        """))
+        await self.db.commit()
+
+    async def get_backup_settings(self) -> Dict[str, Any]:
+        """Get backup configuration from system_config"""
+        settings = {
+            'backup_frequency': 'manual',
+            'backup_retention_count': 7,
+            'backup_destination': 'local',
+            'backup_time': None,
+            'backup_last_run_at': None,
+            'backup_last_status': None
+        }
+
+        result = await self.db.execute(text("""
+            SELECT config_key, config_value
+            FROM system_config
+            WHERE config_key LIKE 'backup_%'
+        """))
+        rows = result.fetchall()
+
+        for row in rows:
+            settings[row.config_key] = row.config_value
+
+        return settings
+
+    async def update_backup_settings(self, updates: Dict[str, Any]) -> bool:
+        """Update backup configuration"""
+        try:
+            for key, value in updates.items():
+                if not key.startswith('backup_'):
+                    key = f'backup_{key}'
+
+                await self.db.execute(text("""
+                    INSERT INTO system_config (config_key, config_value, updated_at)
+                    VALUES (:key, :value, NOW())
+                    ON CONFLICT (config_key) DO UPDATE
+                    SET config_value = :value, updated_at = NOW()
+                """), {'key': key, 'value': str(value)})
+
+            await self.db.commit()
+            return True
+        except Exception as e:
+            logger.error(f"Failed to update backup settings: {e}")
+            await self.db.rollback()
+            return False
+
+    async def create_backup(
+        self,
+        backup_type: str = 'manual',
+        created_by: Optional[str] = None
+    ) -> Tuple[bool, str, Optional[int]]:
+        """
+        Create a full backup of database and files
+
+        Returns: (success, message, backup_id)
+        """
+        backup_id = None
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = f"backup_{timestamp}.zip"
+        filepath = BACKUP_DIR / filename
+
+        try:
+            # Create backup record
+            result = await self.db.execute(text("""
+                INSERT INTO backup_history (
+                    backup_type, status, filename, started_at, created_by
+                ) VALUES (
+                    :backup_type, 'running', :filename, NOW(), :created_by
+                ) RETURNING id
+            """), {
+                'backup_type': backup_type,
+                'filename': filename,
+                'created_by': created_by
+            })
+            await self.db.commit()
+            row = result.fetchone()
+            backup_id = row.id if row else None
+
+            # Create temporary directory for backup contents
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+
+                # 1. Create PostgreSQL dump
+                db_url = os.getenv('DATABASE_URL', 'postgresql://forecast:forecast_secret@localhost:5432/forecast')
+                db_parts = db_url.replace('postgresql://', '').split('@')
+                user_pass = db_parts[0].split(':')
+                host_db = db_parts[1].split('/')
+
+                db_dump_path = temp_path / 'database.sql'
+                env = os.environ.copy()
+                env['PGPASSWORD'] = user_pass[1] if len(user_pass) > 1 else ''
+
+                pg_dump_cmd = [
+                    'pg_dump',
+                    '-h', host_db[0].split(':')[0],
+                    '-U', user_pass[0],
+                    '-d', host_db[1] if len(host_db) > 1 else 'forecast',
+                    '-f', str(db_dump_path),
+                    '--no-owner',
+                    '--no-acl'
+                ]
+
+                subprocess.run(pg_dump_cmd, env=env, check=True, capture_output=True)
+
+                # 2. Create JSON export
+                db_json_path = temp_path / 'database.json'
+                snapshot_count = await self._export_database_json(db_json_path)
+
+                # 3. Copy files
+                files_dir = temp_path / 'files'
+                file_count = self._copy_data_files(files_dir)
+
+                # 4. Create metadata
+                metadata = {
+                    'version': '1.0',
+                    'timestamp': datetime.now().isoformat(),
+                    'snapshot_count': snapshot_count,
+                    'file_count': file_count,
+                    'database_url': db_url.split('@')[1]  # Only host/db, not credentials
+                }
+                metadata_path = temp_path / 'metadata.json'
+                with open(metadata_path, 'w') as f:
+                    json.dump(metadata, f, indent=2)
+
+                # 5. Create ZIP file
+                with zipfile.ZipFile(filepath, 'w', zipfile.ZIP_DEFLATED) as zf:
+                    for file in temp_path.rglob('*'):
+                        if file.is_file():
+                            arcname = file.relative_to(temp_path)
+                            zf.write(file, arcname)
+
+            # Update backup record with success
+            file_size = filepath.stat().st_size
+            await self.db.execute(text("""
+                UPDATE backup_history
+                SET status = 'success',
+                    file_path = :file_path,
+                    file_size_bytes = :file_size,
+                    snapshot_count = :snapshot_count,
+                    file_count = :file_count,
+                    completed_at = NOW()
+                WHERE id = :backup_id
+            """), {
+                'backup_id': backup_id,
+                'file_path': str(filepath),
+                'file_size': file_size,
+                'snapshot_count': snapshot_count,
+                'file_count': file_count
+            })
+            await self.db.commit()
+
+            # Update last backup settings
+            await self.update_backup_settings({
+                'backup_last_run_at': datetime.now().isoformat(),
+                'backup_last_status': 'success'
+            })
+
+            # Enforce retention policy
+            await self._enforce_retention()
+
+            return True, f"Backup created successfully: {filename}", backup_id
+
+        except Exception as e:
+            logger.error(f"Backup creation failed: {e}")
+
+            # Update backup record with failure
+            if backup_id:
+                await self.db.execute(text("""
+                    UPDATE backup_history
+                    SET status = 'failed',
+                        error_message = :error,
+                        completed_at = NOW()
+                    WHERE id = :backup_id
+                """), {
+                    'backup_id': backup_id,
+                    'error': str(e)
+                })
+                await self.db.commit()
+
+            # Update last backup status
+            await self.update_backup_settings({
+                'backup_last_run_at': datetime.now().isoformat(),
+                'backup_last_status': 'failed'
+            })
+
+            return False, f"Backup failed: {str(e)}", backup_id
+
+    async def _export_database_json(self, output_path: Path) -> int:
+        """Export database tables to JSON format"""
+        export_data = {
+            'exported_at': datetime.now().isoformat(),
+            'tables': {}
+        }
+
+        # Tables to export
+        tables = [
+            'forecast_snapshots',
+            'special_dates',
+            'newbook_bookings_data',
+            'newbook_bookings_stats',
+            'newbook_booking_pace',
+            'newbook_occupancy_report_data',
+            'newbook_room_categories',
+            'monthly_budgets',
+            'system_config',
+            'users'
+        ]
+
+        snapshot_count = 0
+        for table in tables:
+            try:
+                result = await self.db.execute(text(f"SELECT * FROM {table}"))
+                rows = result.fetchall()
+                columns = result.keys()
+
+                export_data['tables'][table] = [
+                    {col: self._serialize_value(getattr(row, col)) for col in columns}
+                    for row in rows
+                ]
+
+                if table == 'forecast_snapshots':
+                    snapshot_count = len(rows)
+
+            except Exception as e:
+                logger.warning(f"Could not export table {table}: {e}")
+                export_data['tables'][table] = []
+
+        with open(output_path, 'w') as f:
+            json.dump(export_data, f, indent=2, default=str)
+
+        return snapshot_count
+
+    def _serialize_value(self, value):
+        """Convert value to JSON-serializable format"""
+        if isinstance(value, (datetime,)):
+            return value.isoformat()
+        return value
+
+    def _copy_data_files(self, dest_dir: Path) -> int:
+        """Copy all data files to backup directory"""
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        file_count = 0
+
+        # Skip backup directory itself
+        for root, dirs, files in os.walk(DATA_DIR):
+            # Remove backup directory from traversal
+            dirs[:] = [d for d in dirs if d != 'backups']
+
+            for file in files:
+                src_file = Path(root) / file
+                rel_path = src_file.relative_to(DATA_DIR)
+                dest_file = dest_dir / rel_path
+
+                dest_file.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(src_file, dest_file)
+                file_count += 1
+
+        return file_count
+
+    async def list_backups(self, limit: int = 50) -> list:
+        """List all backups, newest first"""
+        result = await self.db.execute(text("""
+            SELECT
+                id, backup_type, status, filename, file_path,
+                file_size_bytes, snapshot_count, file_count,
+                started_at, completed_at, error_message, created_by
+            FROM backup_history
+            ORDER BY started_at DESC
+            LIMIT :limit
+        """), {'limit': limit})
+
+        rows = result.fetchall()
+        return [
+            {
+                'id': row.id,
+                'backup_type': row.backup_type,
+                'status': row.status,
+                'filename': row.filename,
+                'file_path': row.file_path,
+                'file_size_bytes': row.file_size_bytes,
+                'snapshot_count': row.snapshot_count,
+                'file_count': row.file_count,
+                'started_at': row.started_at.isoformat() if row.started_at else None,
+                'completed_at': row.completed_at.isoformat() if row.completed_at else None,
+                'error_message': row.error_message,
+                'created_by': row.created_by
+            }
+            for row in rows
+        ]
+
+    async def get_backup(self, backup_id: int) -> Optional[Dict]:
+        """Get a specific backup by ID"""
+        result = await self.db.execute(text("""
+            SELECT
+                id, backup_type, status, filename, file_path,
+                file_size_bytes, snapshot_count, file_count,
+                started_at, completed_at, error_message, created_by
+            FROM backup_history
+            WHERE id = :backup_id
+        """), {'backup_id': backup_id})
+
+        row = result.fetchone()
+        if not row:
+            return None
+
+        return {
+            'id': row.id,
+            'backup_type': row.backup_type,
+            'status': row.status,
+            'filename': row.filename,
+            'file_path': row.file_path,
+            'file_size_bytes': row.file_size_bytes,
+            'snapshot_count': row.snapshot_count,
+            'file_count': row.file_count,
+            'started_at': row.started_at.isoformat() if row.started_at else None,
+            'completed_at': row.completed_at.isoformat() if row.completed_at else None,
+            'error_message': row.error_message,
+            'created_by': row.created_by
+        }
+
+    async def delete_backup(self, backup_id: int) -> Tuple[bool, str]:
+        """Delete a backup file and record"""
+        try:
+            # Get backup info
+            backup = await self.get_backup(backup_id)
+            if not backup:
+                return False, "Backup not found"
+
+            # Delete file if it exists
+            if backup['file_path']:
+                file_path = Path(backup['file_path'])
+                if file_path.exists():
+                    file_path.unlink()
+
+            # Delete database record
+            await self.db.execute(text("""
+                DELETE FROM backup_history WHERE id = :backup_id
+            """), {'backup_id': backup_id})
+            await self.db.commit()
+
+            return True, "Backup deleted successfully"
+
+        except Exception as e:
+            logger.error(f"Failed to delete backup: {e}")
+            await self.db.rollback()
+            return False, f"Failed to delete backup: {str(e)}"
+
+    async def restore_from_backup(self, backup_id: int) -> Tuple[bool, str]:
+        """Restore database and files from a backup"""
+        try:
+            # Get backup info
+            backup = await self.get_backup(backup_id)
+            if not backup:
+                return False, "Backup not found"
+
+            if backup['status'] != 'success':
+                return False, "Cannot restore from failed backup"
+
+            backup_path = Path(backup['file_path'])
+            if not backup_path.exists():
+                return False, "Backup file not found"
+
+            return await self._restore_from_file(backup_path)
+
+        except Exception as e:
+            logger.error(f"Restore failed: {e}")
+            return False, f"Restore failed: {str(e)}"
+
+    async def restore_from_upload(self, file_content: bytes, filename: str) -> Tuple[bool, str]:
+        """Restore from an uploaded backup file"""
+        try:
+            # Save uploaded file temporarily
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.zip') as temp_file:
+                temp_file.write(file_content)
+                temp_path = Path(temp_file.name)
+
+            try:
+                # Validate ZIP file
+                if not zipfile.is_zipfile(temp_path):
+                    return False, "Invalid backup file (not a ZIP file)"
+
+                return await self._restore_from_file(temp_path)
+            finally:
+                # Clean up temp file
+                if temp_path.exists():
+                    temp_path.unlink()
+
+        except Exception as e:
+            logger.error(f"Upload restore failed: {e}")
+            return False, f"Restore failed: {str(e)}"
+
+    async def _restore_from_file(self, backup_path: Path) -> Tuple[bool, str]:
+        """Internal method to restore from a backup ZIP file"""
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+
+                # Extract ZIP
+                with zipfile.ZipFile(backup_path, 'r') as zf:
+                    zf.extractall(temp_path)
+
+                # Validate required files
+                db_sql_path = temp_path / 'database.sql'
+                if not db_sql_path.exists():
+                    return False, "Invalid backup: missing database.sql"
+
+                # Restore database
+                db_url = os.getenv('DATABASE_URL', 'postgresql://forecast:forecast_secret@localhost:5432/forecast')
+                db_parts = db_url.replace('postgresql://', '').split('@')
+                user_pass = db_parts[0].split(':')
+                host_db = db_parts[1].split('/')
+
+                env = os.environ.copy()
+                env['PGPASSWORD'] = user_pass[1] if len(user_pass) > 1 else ''
+
+                psql_cmd = [
+                    'psql',
+                    '-h', host_db[0].split(':')[0],
+                    '-U', user_pass[0],
+                    '-d', host_db[1] if len(host_db) > 1 else 'forecast',
+                    '-f', str(db_sql_path)
+                ]
+
+                result = subprocess.run(psql_cmd, env=env, capture_output=True, text=True)
+                if result.returncode != 0:
+                    logger.error(f"Database restore failed: {result.stderr}")
+                    return False, f"Database restore failed: {result.stderr}"
+
+                # Restore files
+                files_dir = temp_path / 'files'
+                if files_dir.exists():
+                    file_count = 0
+                    for root, dirs, files in os.walk(files_dir):
+                        for file in files:
+                            src_file = Path(root) / file
+                            rel_path = src_file.relative_to(files_dir)
+                            dest_file = DATA_DIR / rel_path
+
+                            dest_file.parent.mkdir(parents=True, exist_ok=True)
+                            # Don't overwrite existing files
+                            if not dest_file.exists():
+                                shutil.copy2(src_file, dest_file)
+                                file_count += 1
+
+                    logger.info(f"Restored {file_count} files")
+
+                return True, "Backup restored successfully"
+
+        except Exception as e:
+            logger.error(f"Restore from file failed: {e}")
+            return False, f"Restore failed: {str(e)}"
+
+    async def _enforce_retention(self):
+        """Delete old backups beyond retention count"""
+        try:
+            settings = await self.get_backup_settings()
+            retention_count = int(settings.get('backup_retention_count', 7))
+
+            # Get backups to delete (beyond retention count)
+            result = await self.db.execute(text("""
+                SELECT id, file_path
+                FROM backup_history
+                WHERE status = 'success'
+                ORDER BY started_at DESC
+                OFFSET :retention_count
+            """), {'retention_count': retention_count})
+
+            rows = result.fetchall()
+            for row in rows:
+                # Delete file
+                if row.file_path:
+                    file_path = Path(row.file_path)
+                    if file_path.exists():
+                        file_path.unlink()
+
+                # Delete record
+                await self.db.execute(text("""
+                    DELETE FROM backup_history WHERE id = :backup_id
+                """), {'backup_id': row.id})
+
+            await self.db.commit()
+            logger.info(f"Retention policy: deleted {len(rows)} old backups")
+
+        except Exception as e:
+            logger.error(f"Failed to enforce retention policy: {e}")

backend/services/booking_scraper.py

@@ -0,0 +1,829 @@
+"""
+Booking.com Rate Scraper Service
+
+Main service for scraping competitor rates from booking.com.
+Uses pluggable backends (Playwright local, proxy, Apify) via factory pattern.
+
+Features:
+- Location-based search (1 query = 40+ hotels)
+- Hotel discovery and tier management
+- Rate extraction with availability status
+- Anti-scrape detection and pause/resume
+"""
+
+import logging
+import uuid
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from typing import List, Optional, Dict, Any
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus
+
+logger = logging.getLogger(__name__)
+
+
+def get_scraper_backend(db: Session) -> ScraperBackend:
+    """
+    Factory to get configured scraper backend.
+
+    Reads backend type from system_config and returns appropriate instance.
+    """
+    # Get backend configuration
+    result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'")
+    ).fetchone()
+
+    backend_type = result.config_value if result and result.config_value else 'playwright_local'
+
+    if backend_type == 'playwright_local':
+        return PlaywrightLocalBackend()
+
+    elif backend_type == 'playwright_proxy':
+        # Get proxy config
+        proxy_result = db.execute(
+            text("""
+                SELECT config_key, config_value FROM system_config
+                WHERE config_key IN ('booking_scraper_proxy_url', 'booking_scraper_proxy_username', 'booking_scraper_proxy_password')
+            """)
+        )
+        proxy_config = {row.config_key: row.config_value for row in proxy_result.fetchall()}
+        return PlaywrightLocalBackend(proxy_config=proxy_config)
+
+    elif backend_type == 'apify':
+        # Future: Apify backend
+        raise NotImplementedError("Apify backend not yet implemented")
+
+    else:
+        logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")
+        return PlaywrightLocalBackend()
+
+
+def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
+    """Get the active scrape location configuration."""
+    result = db.execute(
+        text("""
+            SELECT id, location_name, location_search_url, pages_to_scrape, adults
+            FROM booking_scrape_config
+            WHERE is_active = TRUE
+            ORDER BY id
+            LIMIT 1
+        """)
+    ).fetchone()
+
+    if not result:
+        return None
+
+    return {
+        'id': result.id,
+        'location_name': result.location_name,
+        'location_search_url': result.location_search_url,
+        'pages_to_scrape': result.pages_to_scrape or 2,
+        'adults': result.adults or 2,
+    }
+
+
+async def is_scraper_paused(db: Session) -> bool:
+    """Check if scraper is currently paused due to blocking."""
+    result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
+    ).fetchone()
+
+    if not result or result.config_value != 'true':
+        return False
+
+    # Check if pause period has expired
+    pause_until_result = db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'")
+    ).fetchone()
+
+    if pause_until_result and pause_until_result.config_value:
+        try:
+            pause_until = datetime.fromisoformat(pause_until_result.config_value)
+            if datetime.now() >= pause_until:
+                # Pause expired, reset
+                db.execute(
+                    text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
+                )
+                db.commit()
+                return False
+        except ValueError:
+            pass
+
+    return True
+
+
+async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
+    """Set scraper pause status."""
+    db.execute(
+        text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
+        {'val': 'true' if paused else 'false'}
+    )
+    if paused:
+        pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
+        db.execute(
+            text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
+            {'val': pause_until}
+        )
+    db.commit()
+
+
+def save_hotel(db: Session, hotel: HotelData) -> int:
+    """
+    Save or update a hotel in the database.
+
+    Returns the hotel's database ID.
+    """
+    # Check if hotel exists
+    existing = db.execute(
+        text("SELECT id FROM booking_com_hotels WHERE booking_com_id = :bid"),
+        {'bid': hotel.booking_com_id}
+    ).fetchone()
+
+    if existing:
+        # Update last_seen_at and any changed fields
+        db.execute(
+            text("""
+                UPDATE booking_com_hotels SET
+                    name = COALESCE(:name, name),
+                    booking_com_url = COALESCE(:url, booking_com_url),
+                    star_rating = COALESCE(:stars, star_rating),
+                    review_score = COALESCE(:score, review_score),
+                    review_count = COALESCE(:count, review_count),
+                    last_seen_at = NOW()
+                WHERE booking_com_id = :bid
+            """),
+            {
+                'bid': hotel.booking_com_id,
+                'name': hotel.name,
+                'url': hotel.booking_com_url,
+                'stars': float(hotel.star_rating) if hotel.star_rating else None,
+                'score': float(hotel.review_score) if hotel.review_score else None,
+                'count': hotel.review_count,
+            }
+        )
+        return existing.id
+    else:
+        # Insert new hotel (default tier is 'market')
+        result = db.execute(
+            text("""
+                INSERT INTO booking_com_hotels
+                (booking_com_id, name, booking_com_url, star_rating, review_score, review_count, tier)
+                VALUES (:bid, :name, :url, :stars, :score, :count, 'market')
+                RETURNING id
+            """),
+            {
+                'bid': hotel.booking_com_id,
+                'name': hotel.name,
+                'url': hotel.booking_com_url,
+                'stars': float(hotel.star_rating) if hotel.star_rating else None,
+                'score': float(hotel.review_score) if hotel.review_score else None,
+                'count': hotel.review_count,
+            }
+        )
+        return result.fetchone().id
+
+
+def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
+    """Save a rate to the database."""
+    db.execute(
+        text("""
+            INSERT INTO booking_com_rates
+            (hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
+             breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id)
+            VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
+                    :breakfast, :cancel, :prepay, :rooms_left, :batch_id)
+        """),
+        {
+            'hotel_id': hotel_id,
+            'rate_date': rate.rate_date,
+            'status': rate.availability_status.value,
+            'rate': float(rate.rate_gross) if rate.rate_gross else None,
+            'currency': rate.currency,
+            'room_type': rate.room_type,
+            'breakfast': rate.breakfast_included,
+            'cancel': rate.free_cancellation,
+            'prepay': rate.no_prepayment,
+            'rooms_left': rate.rooms_left,
+            'batch_id': str(batch_id),
+        }
+    )
+
+
+def create_scrape_batch(db: Session, scrape_type: str) -> uuid.UUID:
+    """Create a new scrape batch log entry."""
+    batch_id = uuid.uuid4()
+    db.execute(
+        text("""
+            INSERT INTO booking_scrape_log
+            (batch_id, scrape_type, started_at, status)
+            VALUES (:batch_id, :scrape_type, NOW(), 'running')
+        """),
+        {'batch_id': str(batch_id), 'scrape_type': scrape_type}
+    )
+    db.commit()
+    return batch_id
+
+
+def update_scrape_batch(
+    db: Session,
+    batch_id: uuid.UUID,
+    status: str,
+    hotels_found: int = 0,
+    rates_scraped: int = 0,
+    error_message: str = None,
+    blocked: bool = False
+):
+    """Update scrape batch log with results."""
+    db.execute(
+        text("""
+            UPDATE booking_scrape_log SET
+                completed_at = CASE WHEN :status IN ('completed', 'failed', 'blocked') THEN NOW() ELSE NULL END,
+                status = :status,
+                hotels_found = :hotels,
+                rates_scraped = :rates,
+                error_message = :error,
+                blocked_at = CASE WHEN :blocked THEN NOW() ELSE NULL END,
+                resume_after = CASE WHEN :blocked THEN NOW() + INTERVAL '2 hours' ELSE NULL END
+            WHERE batch_id = :batch_id
+        """),
+        {
+            'batch_id': str(batch_id),
+            'status': status,
+            'hotels': hotels_found,
+            'rates': rates_scraped,
+            'error': error_message,
+            'blocked': blocked,
+        }
+    )
+    db.commit()
+
+
+def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
+    """
+    Mark any 'running' scrape batches as 'failed' if they've been running
+    longer than max_age_minutes. This handles orphaned batches from
+    container restarts or crashes.
+    """
+    result = db.execute(
+        text("""
+            UPDATE booking_scrape_log SET
+                status = 'failed',
+                completed_at = NOW(),
+                error_message = 'Interrupted (container restart or timeout)'
+            WHERE status = 'running'
+              AND started_at < NOW() - INTERVAL ':mins minutes'
+            RETURNING batch_id
+        """.replace(':mins', str(int(max_age_minutes))))
+    )
+    cleaned = result.fetchall()
+    db.commit()
+    if cleaned:
+        logger.info(f"Cleaned up {len(cleaned)} stale running scrape batch(es)")
+    return len(cleaned)
+
+
+async def scrape_date(
+    db: Session,
+    rate_date: date,
+    backend: ScraperBackend,
+    config: Dict[str, Any],
+    batch_id: uuid.UUID
+) -> Dict[str, Any]:
+    """
+    Scrape rates for a single date.
+
+    Args:
+        db: Database session
+        rate_date: Date to scrape rates for
+        backend: Scraper backend instance
+        config: Scrape configuration
+        batch_id: Current batch ID
+
+    Returns:
+        Dict with 'success', 'blocked', 'hotels_count', 'rates_count'
+    """
+    check_in = rate_date
+    check_out = rate_date + timedelta(days=1)  # Single night
+
+    result = await backend.scrape_location_search(
+        location=config['location_name'],
+        check_in=check_in,
+        check_out=check_out,
+        adults=config['adults'],
+        pages=config['pages_to_scrape']
+    )
+
+    if result.blocked:
+        return {
+            'success': False,
+            'blocked': True,
+            'block_reason': result.block_reason,
+            'hotels_count': 0,
+            'rates_count': 0,
+        }
+
+    if not result.success:
+        return {
+            'success': False,
+            'blocked': False,
+            'error': result.error_message,
+            'hotels_count': 0,
+            'rates_count': 0,
+        }
+
+    # Save hotels and rates
+    hotels_saved = 0
+    rates_saved = 0
+
+    for hotel, rate in zip(result.hotels, result.rates):
+        if not hotel.booking_com_id:
+            continue
+
+        try:
+            hotel_id = save_hotel(db, hotel)
+            save_rate(db, rate, hotel_id, batch_id)
+            hotels_saved += 1
+            rates_saved += 1
+        except Exception as e:
+            logger.warning(f"Error saving hotel/rate: {e}")
+            continue
+
+    db.commit()
+
+    return {
+        'success': True,
+        'blocked': False,
+        'hotels_count': hotels_saved,
+        'rates_count': rates_saved,
+    }
+
+
+async def run_manual_scrape(
+    db: Session,
+    from_date: date,
+    to_date: date = None
+) -> Dict[str, Any]:
+    """
+    Run a manual scrape for testing/on-demand use.
+
+    Args:
+        db: Database session
+        from_date: Start date
+        to_date: End date (defaults to from_date for single day)
+
+    Returns:
+        Dict with scrape results summary
+    """
+    if to_date is None:
+        to_date = from_date
+
+    # Check if paused
+    if await is_scraper_paused(db):
+        return {
+            'success': False,
+            'error': 'Scraper is currently paused due to blocking. Try again later.',
+        }
+
+    # Get config
+    config = get_scrape_config(db)
+    if not config:
+        return {
+            'success': False,
+            'error': 'No scrape location configured. Add a location in settings.',
+        }
+
+    # Create batch
+    batch_id = create_scrape_batch(db, 'manual')
+
+    # Get backend
+    backend = get_scraper_backend(db)
+
+    total_hotels = 0
+    total_rates = 0
+    dates_completed = 0
+    dates_failed = 0
+
+    try:
+        current_date = from_date
+        while current_date <= to_date:
+            logger.info(f"Scraping date: {current_date}")
+
+            result = await scrape_date(db, current_date, backend, config, batch_id)
+
+            if result['blocked']:
+                # Blocking detected - pause and exit
+                await set_scraper_paused(db, True, hours=2)
+                update_scrape_batch(
+                    db, batch_id,
+                    status='blocked',
+                    hotels_found=total_hotels,
+                    rates_scraped=total_rates,
+                    error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
+                    blocked=True
+                )
+                return {
+                    'success': False,
+                    'blocked': True,
+                    'block_reason': result.get('block_reason'),
+                    'dates_completed': dates_completed,
+                    'dates_failed': dates_failed,
+                    'hotels_found': total_hotels,
+                    'rates_scraped': total_rates,
+                }
+
+            if result['success']:
+                total_hotels += result['hotels_count']
+                total_rates += result['rates_count']
+                dates_completed += 1
+            else:
+                dates_failed += 1
+                logger.warning(f"Failed to scrape {current_date}: {result.get('error')}")
+
+            current_date += timedelta(days=1)
+
+        # Update batch as completed
+        update_scrape_batch(
+            db, batch_id,
+            status='completed',
+            hotels_found=total_hotels,
+            rates_scraped=total_rates
+        )
+
+        return {
+            'success': True,
+            'blocked': False,
+            'dates_completed': dates_completed,
+            'dates_failed': dates_failed,
+            'hotels_found': total_hotels,
+            'rates_scraped': total_rates,
+        }
+
+    except Exception as e:
+        logger.error(f"Scrape error: {e}")
+        update_scrape_batch(
+            db, batch_id,
+            status='failed',
+            hotels_found=total_hotels,
+            rates_scraped=total_rates,
+            error_message=str(e)
+        )
+        return {
+            'success': False,
+            'error': str(e),
+            'dates_completed': dates_completed,
+            'dates_failed': dates_failed,
+            'hotels_found': total_hotels,
+            'rates_scraped': total_rates,
+        }
+    finally:
+        await backend.close()
+
+
+# ============================================
+# QUEUE MANAGEMENT
+# ============================================
+
+def populate_queue(db: Session, dates: List[date], priorities: Dict[date, int] = None):
+    """
+    Add dates to the scrape queue, skipping any already pending/processing.
+
+    Args:
+        db: Database session
+        dates: Dates to add to the queue
+        priorities: Optional priority map (higher = scraped first). Default: 0
+    """
+    if not dates:
+        return 0
+
+    added = 0
+    for rate_date in dates:
+        priority = (priorities or {}).get(rate_date, 0)
+        try:
+            db.execute(
+                text("""
+                    INSERT INTO booking_scrape_queue (rate_date, status, priority)
+                    VALUES (:rate_date, 'pending', :priority)
+                    ON CONFLICT (rate_date, status) DO UPDATE SET
+                        priority = GREATEST(booking_scrape_queue.priority, :priority)
+                """),
+                {'rate_date': rate_date, 'priority': priority}
+            )
+            added += 1
+        except Exception:
+            # Ignore duplicates or constraint issues
+            pass
+
+    db.commit()
+    logger.info(f"Queue: added/updated {added} dates")
+    return added
+
+
+def get_pending_queue_items(db: Session, limit: int = 50) -> List[Dict[str, Any]]:
+    """Get pending queue items ordered by priority (highest first), then date."""
+    result = db.execute(
+        text("""
+            SELECT id, rate_date, priority, attempts, max_attempts
+            FROM booking_scrape_queue
+            WHERE status = 'pending' AND attempts < max_attempts
+            ORDER BY priority DESC, rate_date ASC
+            LIMIT :limit
+        """),
+        {'limit': limit}
+    )
+    return [dict(row._mapping) for row in result.fetchall()]
+
+
+def mark_queue_item(db: Session, queue_id: int, status: str, error_message: str = None):
+    """Update a queue item's status."""
+    if status == 'completed':
+        db.execute(
+            text("""
+                UPDATE booking_scrape_queue SET
+                    status = 'completed',
+                    completed_at = NOW(),
+                    last_attempt_at = NOW(),
+                    attempts = attempts + 1
+                WHERE id = :id
+            """),
+            {'id': queue_id}
+        )
+    elif status == 'failed':
+        db.execute(
+            text("""
+                UPDATE booking_scrape_queue SET
+                    status = CASE
+                        WHEN attempts + 1 >= max_attempts THEN 'failed'
+                        ELSE 'pending'
+                    END,
+                    last_attempt_at = NOW(),
+                    attempts = attempts + 1,
+                    error_message = :error
+                WHERE id = :id
+            """),
+            {'id': queue_id, 'error': error_message}
+        )
+    db.commit()
+
+
+def clear_old_queue_items(db: Session, days: int = 7):
+    """Remove completed/failed queue items older than N days."""
+    db.execute(
+        text("""
+            DELETE FROM booking_scrape_queue
+            WHERE status IN ('completed', 'failed')
+              AND created_at < NOW() - INTERVAL ':days days'
+        """.replace(':days', str(int(days))))
+    )
+    db.commit()
+
+
+async def process_queue(db: Session) -> Dict[str, Any]:
+    """
+    Process pending items from the scrape queue.
+
+    Picks up pending items in priority order, scrapes each date,
+    and handles blocking/retries.
+
+    Returns:
+        Dict with processing results
+    """
+    # Check if paused
+    if await is_scraper_paused(db):
+        return {
+            'success': False,
+            'error': 'Scraper is currently paused due to blocking.',
+        }
+
+    # Get config
+    config = get_scrape_config(db)
+    if not config:
+        return {
+            'success': False,
+            'error': 'No scrape location configured.',
+        }
+
+    # Get pending items
+    items = get_pending_queue_items(db, limit=200)
+    if not items:
+        return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'}
+
+    # Create batch
+    batch_id = create_scrape_batch(db, 'scheduled')
+
+    # Update batch with queue count
+    db.execute(
+        text("UPDATE booking_scrape_log SET dates_queued = :count WHERE batch_id = :bid"),
+        {'count': len(items), 'bid': str(batch_id)}
+    )
+    db.commit()
+
+    # Get backend
+    backend = get_scraper_backend(db)
+
+    total_hotels = 0
+    total_rates = 0
+    dates_completed = 0
+    dates_failed = 0
+
+    try:
+        for item in items:
+            rate_date = item['rate_date']
+            queue_id = item['id']
+
+            logger.info(f"Queue processing: {rate_date} (priority={item['priority']}, attempt={item['attempts']+1})")
+
+            result = await scrape_date(db, rate_date, backend, config, batch_id)
+
+            if result['blocked']:
+                # Mark this item as failed, pause, and stop
+                mark_queue_item(db, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
+                await set_scraper_paused(db, True, hours=2)
+                update_scrape_batch(
+                    db, batch_id,
+                    status='blocked',
+                    hotels_found=total_hotels,
+                    rates_scraped=total_rates,
+                    error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
+                    blocked=True
+                )
+                # Update dates counters
+                db.execute(
+                    text("""
+                        UPDATE booking_scrape_log SET
+                            dates_completed = :completed,
+                            dates_failed = :failed
+                        WHERE batch_id = :bid
+                    """),
+                    {'completed': dates_completed, 'failed': dates_failed + 1, 'bid': str(batch_id)}
+                )
+                db.commit()
+                return {
+                    'success': False,
+                    'blocked': True,
+                    'block_reason': result.get('block_reason'),
+                    'dates_completed': dates_completed,
+                    'dates_failed': dates_failed + 1,
+                    'hotels_found': total_hotels,
+                    'rates_scraped': total_rates,
+                }
+
+            if result['success']:
+                mark_queue_item(db, queue_id, 'completed')
+                total_hotels += result['hotels_count']
+                total_rates += result['rates_count']
+                dates_completed += 1
+            else:
+                mark_queue_item(db, queue_id, 'failed', result.get('error'))
+                dates_failed += 1
+                logger.warning(f"Queue: failed to scrape {rate_date}: {result.get('error')}")
+
+        # Update batch as completed
+        update_scrape_batch(
+            db, batch_id,
+            status='completed',
+            hotels_found=total_hotels,
+            rates_scraped=total_rates
+        )
+        db.execute(
+            text("""
+                UPDATE booking_scrape_log SET
+                    dates_completed = :completed,
+                    dates_failed = :failed
+                WHERE batch_id = :bid
+            """),
+            {'completed': dates_completed, 'failed': dates_failed, 'bid': str(batch_id)}
+        )
+        db.commit()
+
+        return {
+            'success': True,
+            'blocked': False,
+            'dates_completed': dates_completed,
+            'dates_failed': dates_failed,
+            'hotels_found': total_hotels,
+            'rates_scraped': total_rates,
+        }
+
+    except Exception as e:
+        logger.error(f"Queue processing error: {e}")
+        update_scrape_batch(
+            db, batch_id,
+            status='failed',
+            hotels_found=total_hotels,
+            rates_scraped=total_rates,
+            error_message=str(e)
+        )
+        return {
+            'success': False,
+            'error': str(e),
+            'dates_completed': dates_completed,
+            'dates_failed': dates_failed,
+        }
+    finally:
+        await backend.close()
+
+
+def get_competitor_matrix(
+    db: Session,
+    from_date: date,
+    to_date: date,
+    include_market: bool = False
+) -> List[Dict[str, Any]]:
+    """
+    Get rate comparison matrix for competitors.
+
+    Args:
+        db: Database session
+        from_date: Start date
+        to_date: End date
+        include_market: Include market tier hotels
+
+    Returns:
+        List of rate records for matrix display
+    """
+    tier_filter = "h.tier IN ('own', 'competitor')"
+    if include_market:
+        tier_filter = "h.tier IN ('own', 'competitor', 'market')"
+
+    result = db.execute(
+        text(f"""
+            SELECT
+                r.rate_date,
+                h.id AS hotel_id,
+                h.name AS hotel_name,
+                h.tier,
+                h.display_order,
+                h.star_rating,
+                h.review_score,
+                r.availability_status,
+                r.rate_gross,
+                r.room_type,
+                r.breakfast_included,
+                r.free_cancellation,
+                r.no_prepayment,
+                r.rooms_left,
+                r.scraped_at
+            FROM booking_latest_rates r
+            JOIN booking_com_hotels h ON r.hotel_id = h.id
+            WHERE {tier_filter}
+              AND h.is_active = TRUE
+              AND r.rate_date BETWEEN :from_date AND :to_date
+            ORDER BY r.rate_date, h.display_order, h.name
+        """),
+        {'from_date': from_date, 'to_date': to_date}
+    )
+
+    return [dict(row._mapping) for row in result.fetchall()]
+
+
+def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
+    """
+    Get list of discovered hotels.
+
+    Args:
+        db: Database session
+        tier: Filter by tier ('own', 'competitor', 'market') or None for all
+
+    Returns:
+        List of hotel records
+    """
+    where_clause = "WHERE is_active = TRUE"
+    if tier:
+        where_clause += f" AND tier = '{tier}'"
+
+    result = db.execute(
+        text(f"""
+            SELECT
+                id, booking_com_id, name, booking_com_url,
+                star_rating, review_score, review_count,
+                tier, display_order, notes,
+                first_seen_at, last_seen_at
+            FROM booking_com_hotels
+            {where_clause}
+            ORDER BY display_order, name
+        """)
+    )
+
+    return [dict(row._mapping) for row in result.fetchall()]
+
+
+def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
+    """Update a hotel's tier and display order."""
+    if tier not in ('own', 'competitor', 'market'):
+        raise ValueError(f"Invalid tier: {tier}")
+
+    params = {'hotel_id': hotel_id, 'tier': tier}
+    set_clause = "tier = :tier"
+
+    if display_order is not None:
+        set_clause += ", display_order = :order"
+        params['order'] = display_order
+
+    db.execute(
+        text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
+        params
+    )
+    db.commit()

backend/services/forecasting/__init__.py

@@ -0,0 +1 @@
+# Forecasting models

backend/services/forecasting/backtest.py

@@ -0,0 +1,401 @@
+"""
+Backtesting service for forecast model evaluation.
+
+Simulates historical forecasts using only data that would have been
+available at the time, then compares to actual outcomes.
+
+This allows model accuracy evaluation without waiting for real-time
+data to accumulate.
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional, Dict
+from sqlalchemy import text
+
+from utils.time_alignment import get_prior_year_daily
+
+logger = logging.getLogger(__name__)
+
+
+async def run_backtest(
+    db,
+    metric_code: str,
+    backtest_from: date,
+    backtest_to: date,
+    lead_times: List[int] = None
+) -> Dict:
+    """
+    Run backtesting for a metric over a date range.
+
+    For each date in the range, simulates what the forecast would have been
+    at various lead times, using only data available at that time.
+
+    Args:
+        db: Database session
+        metric_code: Metric to backtest (e.g., 'hotel_room_nights')
+        backtest_from: Start of backtest period
+        backtest_to: End of backtest period
+        lead_times: List of lead times to test (days out). Default: [7, 14, 21, 28]
+
+    Returns:
+        Dict with backtest results and accuracy metrics
+    """
+    if lead_times is None:
+        lead_times = [7, 14, 21, 28]
+
+    logger.info(f"Running backtest for {metric_code} from {backtest_from} to {backtest_to}")
+
+    results = []
+    total_rooms = 25  # Default capacity
+
+    # Get room capacity (SUM across all room categories for a single date)
+    if metric_code in ('hotel_room_nights', 'hotel_occupancy_pct'):
+        rooms_result = db.execute(
+            text("""
+            SELECT COALESCE(SUM(available), 25) as total_rooms
+            FROM newbook_occupancy_report
+            WHERE date = (
+                SELECT MAX(date) FROM newbook_occupancy_report
+                WHERE date <= :from_date
+            )
+            """),
+            {"from_date": backtest_from}
+        )
+        rooms_row = rooms_result.fetchone()
+        if rooms_row and rooms_row.total_rooms:
+            total_rooms = int(rooms_row.total_rooms)
+
+    # For each date in backtest range
+    current_date = backtest_from
+    while current_date <= backtest_to:
+        # Get actual value for this date
+        actual_result = db.execute(
+            text("""
+            SELECT actual_value
+            FROM daily_metrics
+            WHERE date = :target_date AND metric_code = :metric
+            """),
+            {"target_date": current_date, "metric": metric_code}
+        ).fetchone()
+
+        actual_value = float(actual_result.actual_value) if actual_result and actual_result.actual_value else None
+
+        if actual_value is None:
+            current_date += timedelta(days=1)
+            continue
+
+        # For each lead time, simulate the forecast
+        for lead_time in lead_times:
+            # The "simulated today" is lead_time days before the target date
+            simulated_today = current_date - timedelta(days=lead_time)
+
+            # Get OTB snapshot that would have been available
+            # Look for snapshot closest to simulated_today
+            otb_result = db.execute(
+                text("""
+                SELECT otb_value, snapshot_date, days_out
+                FROM pickup_snapshots
+                WHERE stay_date = :target_date
+                    AND metric_type = :metric
+                    AND snapshot_date <= :simulated_today
+                ORDER BY snapshot_date DESC
+                LIMIT 1
+                """),
+                {
+                    "target_date": current_date,
+                    "metric": metric_code,
+                    "simulated_today": simulated_today
+                }
+            ).fetchone()
+
+            if not otb_result:
+                continue
+
+            # Use 'is not None' - 0 is valid OTB data
+            current_otb = float(otb_result.otb_value) if otb_result.otb_value is not None else 0
+            actual_lead_time = otb_result.days_out or lead_time
+
+            # Get prior year comparison data (same day of week)
+            prior_year_date = get_prior_year_daily(current_date)
+            prior_year_simulated_today = get_prior_year_daily(simulated_today)
+
+            # Get prior year OTB at same lead time
+            prior_otb_result = db.execute(
+                text("""
+                SELECT otb_value
+                FROM pickup_snapshots
+                WHERE stay_date = :prior_date
+                    AND metric_type = :metric
+                    AND snapshot_date <= :prior_simulated_today
+                ORDER BY snapshot_date DESC
+                LIMIT 1
+                """),
+                {
+                    "prior_date": prior_year_date,
+                    "metric": metric_code,
+                    "prior_simulated_today": prior_year_simulated_today
+                }
+            ).fetchone()
+
+            # Use 'is not None' - 0 is valid OTB data
+            prior_otb = float(prior_otb_result.otb_value) if prior_otb_result and prior_otb_result.otb_value is not None else None
+
+            # Get prior year final actual
+            prior_final_result = db.execute(
+                text("""
+                SELECT actual_value
+                FROM daily_metrics
+                WHERE date = :prior_date AND metric_code = :metric
+                """),
+                {"prior_date": prior_year_date, "metric": metric_code}
+            ).fetchone()
+
+            prior_final = float(prior_final_result.actual_value) if prior_final_result and prior_final_result.actual_value else None
+
+            # Calculate forecast using ADDITIVE method
+            projected_value = current_otb
+            projection_method = 'current_otb'
+
+            if prior_otb is not None and prior_final is not None:
+                # Additive method: current + expected pickup
+                prior_pickup = prior_final - prior_otb
+                projected_value = current_otb + prior_pickup
+
+                # Floor at current OTB
+                if projected_value < current_otb:
+                    projected_value = current_otb
+                    projection_method = 'additive_floor'
+                else:
+                    projection_method = 'additive'
+
+                # Apply physical caps
+                if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                    projected_value = 100
+                if metric_code == 'hotel_room_nights' and projected_value > total_rooms:
+                    projected_value = total_rooms
+
+            elif prior_final is not None and prior_final > 0:
+                # Implied additive method
+                if lead_time >= 28:
+                    estimated_pct = 0.35
+                elif lead_time >= 14:
+                    estimated_pct = 0.55
+                elif lead_time >= 7:
+                    estimated_pct = 0.75
+                else:
+                    estimated_pct = 0.90
+
+                implied_prior_otb = prior_final * estimated_pct
+                implied_pickup = prior_final - implied_prior_otb
+                projected_value = current_otb + implied_pickup
+                projected_value = max(projected_value, current_otb)
+                projection_method = 'implied_additive'
+
+                # Apply caps
+                if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                    projected_value = 100
+                if metric_code == 'hotel_room_nights' and projected_value > total_rooms:
+                    projected_value = total_rooms
+
+            # Calculate error metrics
+            error = projected_value - actual_value
+            abs_error = abs(error)
+            pct_error = (error / actual_value * 100) if actual_value != 0 else None
+            abs_pct_error = abs(pct_error) if pct_error is not None else None
+
+            result_record = {
+                "target_date": current_date,
+                "lead_time": lead_time,
+                "actual_lead_time": actual_lead_time,
+                "simulated_today": simulated_today,
+                "current_otb": current_otb,
+                "prior_otb": prior_otb,
+                "prior_final": prior_final,
+                "projected_value": round(projected_value, 2),
+                "actual_value": actual_value,
+                "error": round(error, 2),
+                "abs_error": round(abs_error, 2),
+                "pct_error": round(pct_error, 2) if pct_error is not None else None,
+                "abs_pct_error": round(abs_pct_error, 2) if abs_pct_error is not None else None,
+                "projection_method": projection_method
+            }
+            results.append(result_record)
+
+            # Store in backtest_results table
+            try:
+                db.execute(
+                    text("""
+                    INSERT INTO backtest_results (
+                        target_date, metric_code, lead_time, simulated_today,
+                        current_otb, prior_otb, prior_final,
+                        projected_value, actual_value,
+                        error, abs_error, pct_error, abs_pct_error,
+                        projection_method, created_at
+                    ) VALUES (
+                        :target_date, :metric, :lead_time, :simulated_today,
+                        :current_otb, :prior_otb, :prior_final,
+                        :projected_value, :actual_value,
+                        :error, :abs_error, :pct_error, :abs_pct_error,
+                        :projection_method, NOW()
+                    )
+                    ON CONFLICT (target_date, metric_code, lead_time) DO UPDATE SET
+                        projected_value = :projected_value,
+                        actual_value = :actual_value,
+                        error = :error,
+                        abs_error = :abs_error,
+                        pct_error = :pct_error,
+                        abs_pct_error = :abs_pct_error,
+                        projection_method = :projection_method,
+                        created_at = NOW()
+                    """),
+                    {
+                        "target_date": current_date,
+                        "metric": metric_code,
+                        "lead_time": lead_time,
+                        "simulated_today": simulated_today,
+                        "current_otb": current_otb,
+                        "prior_otb": prior_otb,
+                        "prior_final": prior_final,
+                        "projected_value": round(projected_value, 2),
+                        "actual_value": actual_value,
+                        "error": round(error, 2),
+                        "abs_error": round(abs_error, 2),
+                        "pct_error": round(pct_error, 2) if pct_error is not None else None,
+                        "abs_pct_error": round(abs_pct_error, 2) if abs_pct_error is not None else None,
+                        "projection_method": projection_method
+                    }
+                )
+            except Exception as e:
+                logger.warning(f"Could not store backtest result: {e}")
+
+        current_date += timedelta(days=1)
+
+    db.commit()
+
+    # Calculate summary statistics
+    summary = calculate_backtest_summary(results, lead_times)
+
+    logger.info(f"Backtest complete: {len(results)} forecasts evaluated")
+
+    return {
+        "metric_code": metric_code,
+        "backtest_from": str(backtest_from),
+        "backtest_to": str(backtest_to),
+        "lead_times": lead_times,
+        "total_forecasts": len(results),
+        "results": results,
+        "summary": summary
+    }
+
+
+def calculate_backtest_summary(results: List[dict], lead_times: List[int]) -> Dict:
+    """
+    Calculate summary accuracy metrics from backtest results.
+    """
+    if not results:
+        return {}
+
+    summary = {
+        "overall": {},
+        "by_lead_time": {}
+    }
+
+    # Overall metrics
+    all_errors = [r['abs_error'] for r in results if r['abs_error'] is not None]
+    all_pct_errors = [r['abs_pct_error'] for r in results if r['abs_pct_error'] is not None]
+
+    if all_errors:
+        summary["overall"] = {
+            "mae": round(sum(all_errors) / len(all_errors), 2),  # Mean Absolute Error
+            "mape": round(sum(all_pct_errors) / len(all_pct_errors), 2) if all_pct_errors else None,  # Mean Absolute Percentage Error
+            "count": len(all_errors)
+        }
+
+    # By lead time
+    for lt in lead_times:
+        lt_results = [r for r in results if r['lead_time'] == lt]
+        lt_errors = [r['abs_error'] for r in lt_results if r['abs_error'] is not None]
+        lt_pct_errors = [r['abs_pct_error'] for r in lt_results if r['abs_pct_error'] is not None]
+
+        if lt_errors:
+            summary["by_lead_time"][lt] = {
+                "mae": round(sum(lt_errors) / len(lt_errors), 2),
+                "mape": round(sum(lt_pct_errors) / len(lt_pct_errors), 2) if lt_pct_errors else None,
+                "count": len(lt_errors)
+            }
+
+    # By projection method
+    methods = set(r['projection_method'] for r in results)
+    summary["by_method"] = {}
+    for method in methods:
+        method_results = [r for r in results if r['projection_method'] == method]
+        method_errors = [r['abs_error'] for r in method_results if r['abs_error'] is not None]
+        method_pct_errors = [r['abs_pct_error'] for r in method_results if r['abs_pct_error'] is not None]
+
+        if method_errors:
+            summary["by_method"][method] = {
+                "mae": round(sum(method_errors) / len(method_errors), 2),
+                "mape": round(sum(method_pct_errors) / len(method_pct_errors), 2) if method_pct_errors else None,
+                "count": len(method_errors)
+            }
+
+    return summary
+
+
+async def get_backtest_results(
+    db,
+    metric_code: str,
+    from_date: Optional[date] = None,
+    to_date: Optional[date] = None,
+    lead_time: Optional[int] = None
+) -> List[dict]:
+    """
+    Retrieve stored backtest results.
+    """
+    query = """
+    SELECT
+        target_date, metric_code, lead_time, simulated_today,
+        current_otb, prior_otb, prior_final,
+        projected_value, actual_value,
+        error, abs_error, pct_error, abs_pct_error,
+        projection_method, created_at
+    FROM backtest_results
+    WHERE metric_code = :metric
+    """
+    params = {"metric": metric_code}
+
+    if from_date:
+        query += " AND target_date >= :from_date"
+        params["from_date"] = from_date
+
+    if to_date:
+        query += " AND target_date <= :to_date"
+        params["to_date"] = to_date
+
+    if lead_time:
+        query += " AND lead_time = :lead_time"
+        params["lead_time"] = lead_time
+
+    query += " ORDER BY target_date, lead_time"
+
+    result = db.execute(text(query), params)
+
+    return [
+        {
+            "target_date": str(row.target_date),
+            "metric_code": row.metric_code,
+            "lead_time": row.lead_time,
+            "simulated_today": str(row.simulated_today) if row.simulated_today else None,
+            "current_otb": float(row.current_otb) if row.current_otb is not None else None,
+            "prior_otb": float(row.prior_otb) if row.prior_otb is not None else None,
+            "prior_final": float(row.prior_final) if row.prior_final is not None else None,
+            "projected_value": float(row.projected_value) if row.projected_value is not None else None,
+            "actual_value": float(row.actual_value) if row.actual_value is not None else None,
+            "error": float(row.error) if row.error is not None else None,
+            "abs_error": float(row.abs_error) if row.abs_error is not None else None,
+            "pct_error": float(row.pct_error) if row.pct_error is not None else None,
+            "abs_pct_error": float(row.abs_pct_error) if row.abs_pct_error is not None else None,
+            "projection_method": row.projection_method
+        }
+        for row in result.fetchall()
+    ]

backend/services/forecasting/blended_model.py

@@ -0,0 +1,159 @@
+"""
+Centralized Blended Forecasting Model Service
+
+This is the single source of truth for blended forecasts.
+Used by:
+- Weekly snapshots (saves to DB)
+- Frontend live previews (on-the-fly)
+- External apps (reads saved snapshots)
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Dict, Optional
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+async def run_blended_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    save_to_db: bool = False,
+    run_id: Optional[str] = None
+) -> List[Dict]:
+    """
+    Generate blended forecast by running Prophet, XGBoost, CatBoost and blending with accuracy weights.
+
+    This is the centralized blended model used everywhere in the application.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast (e.g., 'hotel_occupancy_pct', 'hotel_room_nights')
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        save_to_db: If True, saves forecasts to database (for snapshots)
+        run_id: Run ID for tracking (required if save_to_db=True)
+
+    Returns:
+        List of forecast dicts with date and predicted_value
+    """
+    logger.info(f"Running blended forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Use simple equal-weight averaging to match frontend behavior
+    # Frontend: (prophet + xgboost + catboost) / 3
+    logger.info(f"Using simple equal-weight averaging for {metric_code}")
+
+    # Step 1: Run individual models
+    forecasts_by_date = {}
+
+    # Run Prophet
+    try:
+        from services.forecasting.prophet_model import run_prophet_forecast
+        prophet_forecasts = await run_prophet_forecast(db, metric_code, start_date, end_date)
+        for fc in prophet_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['prophet'] = float(fc['predicted_value'])
+        logger.info(f"Prophet generated {len(prophet_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"Prophet forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run XGBoost
+    try:
+        from services.forecasting.xgboost_model import run_xgboost_forecast
+        xgboost_forecasts = await run_xgboost_forecast(db, metric_code, start_date, end_date)
+        for fc in xgboost_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['xgboost'] = float(fc['predicted_value'])
+        logger.info(f"XGBoost generated {len(xgboost_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"XGBoost forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run CatBoost
+    try:
+        from services.forecasting.catboost_model import run_catboost_forecast
+        catboost_forecasts = await run_catboost_forecast(db, metric_code, start_date, end_date)
+        for fc in catboost_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['catboost'] = float(fc['predicted_value'])
+        logger.info(f"CatBoost generated {len(catboost_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"CatBoost forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run Pickup-V2 for revenue metrics (net_accom, hotel_accommodation_rev)
+    if metric_code in ('net_accom', 'hotel_accommodation_rev'):
+        try:
+            from services.forecasting.pickup_v2_model import run_pickup_v2_forecast
+            pickup_v2_forecasts = await run_pickup_v2_forecast(db, 'net_accom', start_date, end_date)
+            for fc in pickup_v2_forecasts:
+                fc_date = str(fc['date'])
+                if fc_date not in forecasts_by_date:
+                    forecasts_by_date[fc_date] = {}
+                forecasts_by_date[fc_date]['pickup_v2'] = float(fc['predicted_value'])
+            logger.info(f"Pickup-V2 generated {len(pickup_v2_forecasts)} forecasts for {metric_code}")
+            db.commit()
+        except Exception as e:
+            logger.error(f"Pickup-V2 forecast failed for {metric_code}: {e}")
+            db.rollback()
+            db.commit()
+
+    # Step 3: Calculate simple average blended forecast
+    # Match frontend logic: (prophet + xgboost + catboost) / 3
+    blended_forecasts = []
+    for fc_date, model_forecasts in forecasts_by_date.items():
+        # Need at least 2 models to blend
+        if len(model_forecasts) < 2:
+            continue
+
+        # Simple average of all available models
+        blended_value = sum(model_forecasts.values()) / len(model_forecasts)
+
+        blended_forecasts.append({
+            'date': fc_date,
+            'predicted_value': round(blended_value, 2)
+        })
+
+    logger.info(f"Generated {len(blended_forecasts)} blended forecasts for {metric_code}")
+
+    # Step 4: Optionally save to database (for snapshots)
+    if save_to_db and run_id:
+        try:
+            for fc in blended_forecasts:
+                db.execute(
+                    text("""
+                        INSERT INTO forecasts
+                            (run_id, forecast_date, forecast_type, model_type, predicted_value, generated_at)
+                        VALUES
+                            (:run_id, :forecast_date, :forecast_type, 'blended', :predicted_value, NOW())
+                    """),
+                    {
+                        "run_id": run_id,
+                        "forecast_date": fc['date'],
+                        "forecast_type": metric_code,
+                        "predicted_value": fc['predicted_value']
+                    }
+                )
+            db.commit()
+            logger.info(f"Saved {len(blended_forecasts)} blended forecasts to database")
+        except Exception as e:
+            logger.error(f"Failed to save blended forecasts to database: {e}")
+            db.rollback()
+            raise
+
+    return blended_forecasts

backend/services/forecasting/blended_tuned.py

@@ -0,0 +1,164 @@
+"""
+Blended Tuned Model Service
+
+This blends the production-tuned models using the exact same logic as the frontend.
+Ensures backend snapshots match frontend preview values.
+
+Blending Logic:
+- Pace metrics (rooms/occupancy): Prophet + XGBoost + CatBoost + Pickup (25% each)
+- Other metrics: Prophet + XGBoost + CatBoost (33.3% each)
+
+This is the single source of truth for blended forecast snapshots.
+"""
+import logging
+from datetime import date
+from typing import List, Dict, Optional
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+async def run_blended_tuned_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    save_to_db: bool = False,
+    run_id: Optional[str] = None
+) -> List[Dict]:
+    """
+    Generate blended forecast using production-tuned models.
+
+    This uses the exact same logic as the frontend Live Blended view to ensure
+    backend snapshots match frontend preview values.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast (e.g., 'hotel_occupancy_pct', 'hotel_room_nights')
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        save_to_db: If True, saves forecasts to database (for snapshots)
+        run_id: Run ID for tracking (required if save_to_db=True)
+
+    Returns:
+        List of forecast dicts with date and predicted_value
+    """
+    logger.info(f"Running blended tuned forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Check if metric is a pace metric (uses pickup)
+    is_pace_metric = metric_code in ('hotel_occupancy_pct', 'hotel_room_nights')
+
+    # Step 1: Run individual tuned models
+    forecasts_by_date = {}
+
+    # Run Prophet Tuned
+    try:
+        from services.forecasting.prophet_tuned import run_prophet_tuned_forecast
+        prophet_forecasts = await run_prophet_tuned_forecast(db, metric_code, start_date, end_date)
+        for fc in prophet_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['prophet'] = float(fc['predicted_value'])
+        logger.info(f"Prophet tuned generated {len(prophet_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"Prophet tuned forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run XGBoost Tuned
+    try:
+        from services.forecasting.xgboost_tuned import run_xgboost_tuned_forecast
+        xgboost_forecasts = await run_xgboost_tuned_forecast(db, metric_code, start_date, end_date)
+        for fc in xgboost_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['xgboost'] = float(fc['predicted_value'])
+        logger.info(f"XGBoost tuned generated {len(xgboost_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"XGBoost tuned forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run CatBoost Tuned
+    try:
+        from services.forecasting.catboost_tuned import run_catboost_tuned_forecast
+        catboost_forecasts = await run_catboost_tuned_forecast(db, metric_code, start_date, end_date)
+        for fc in catboost_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['catboost'] = float(fc['predicted_value'])
+        logger.info(f"CatBoost tuned generated {len(catboost_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"CatBoost tuned forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run Pickup Tuned (only for pace metrics)
+    if is_pace_metric:
+        try:
+            from services.forecasting.pickup_tuned import run_pickup_tuned_forecast
+            pickup_forecasts = await run_pickup_tuned_forecast(db, metric_code, start_date, end_date)
+            for fc in pickup_forecasts:
+                fc_date = str(fc['forecast_date'])
+                if fc_date not in forecasts_by_date:
+                    forecasts_by_date[fc_date] = {}
+                forecasts_by_date[fc_date]['pickup'] = float(fc['predicted_value'])
+            logger.info(f"Pickup tuned generated {len(pickup_forecasts)} forecasts for {metric_code}")
+            db.commit()  # Commit after successful model run
+        except Exception as e:
+            logger.error(f"Pickup tuned forecast failed for {metric_code}: {e}")
+            db.rollback()
+            db.commit()  # Start fresh transaction
+
+    # Step 2: Calculate simple average blended forecast
+    # Match frontend logic:
+    # - Pace metrics: (prophet + xgboost + catboost + pickup) / 4
+    # - Other metrics: (prophet + xgboost + catboost) / 3
+    blended_forecasts = []
+    for fc_date, model_forecasts in forecasts_by_date.items():
+        # Need at least 2 models to blend
+        if len(model_forecasts) < 2:
+            continue
+
+        # Simple average of all available models
+        blended_value = sum(model_forecasts.values()) / len(model_forecasts)
+
+        blended_forecasts.append({
+            'date': fc_date,
+            'predicted_value': round(blended_value, 2)
+        })
+
+    logger.info(f"Generated {len(blended_forecasts)} blended tuned forecasts for {metric_code}")
+
+    # Step 3: Optionally save to database (for snapshots)
+    if save_to_db and run_id:
+        try:
+            for fc in blended_forecasts:
+                db.execute(
+                    text("""
+                        INSERT INTO forecasts
+                            (run_id, forecast_date, forecast_type, model_type, predicted_value, generated_at)
+                        VALUES
+                            (:run_id, :forecast_date, :forecast_type, 'blended_tuned', :predicted_value, NOW())
+                    """),
+                    {
+                        "run_id": run_id,
+                        "forecast_date": fc['date'],
+                        "forecast_type": metric_code,
+                        "predicted_value": fc['predicted_value']
+                    }
+                )
+            db.commit()
+            logger.info(f"Saved {len(blended_forecasts)} blended tuned forecasts to database")
+        except Exception as e:
+            logger.error(f"Failed to save blended tuned forecasts to database: {e}")
+            db.rollback()
+            raise
+
+    return blended_forecasts

backend/services/forecasting/blended_tuned_weighted.py

@@ -0,0 +1,336 @@
+"""
+Blended Tuned Model Service (Accuracy-Weighted + 60/40 Prior Year/Budget)
+
+Two-stage blending:
+1. MAPE-weighted model blend using backtest accuracy data
+2. 60/40 blend with prior year actual or budget
+
+Stage 1 - Model Weighting (MAPE-based):
+- Query forecast_snapshots table for backtest MAPE scores
+- Calculate inverse-MAPE weights (lower MAPE = higher weight)
+- Pace metrics: weighted blend of Prophet + XGBoost + CatBoost + Pickup
+- Other metrics: weighted blend of Prophet + XGBoost + CatBoost
+
+Stage 2 - 60/40 Blend:
+- Revenue metrics: 60% weighted model blend + 40% budget
+- Non-revenue metrics: 60% weighted model blend + 40% prior year actual
+
+Falls back to 100% model blend if prior year/budget data unavailable.
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Dict, Optional
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+async def get_model_weights(db, metric_code: str, is_pace_metric: bool) -> Dict[str, float]:
+    """
+    Calculate accuracy-based weights for each model using MAPE scores from backtest data.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast
+        is_pace_metric: Whether this is a pace metric (uses pickup)
+
+    Returns:
+        Dict of model names to weights (normalized to sum to 1.0)
+    """
+    # Map metric codes to forecast_snapshots metric codes
+    metric_map = {
+        'hotel_occupancy_pct': 'occupancy',
+        'hotel_room_nights': 'rooms',
+        'hotel_guests': 'guests',
+        'hotel_arr': 'arr',
+        'ave_guest_rate': 'ave_guest_rate',
+        'net_accom': 'net_accom',
+        'net_dry': 'net_dry',
+        'net_wet': 'net_wet',
+        'total_rev': 'total_rev',
+    }
+
+    snapshot_metric = metric_map.get(metric_code, 'rooms')
+
+    try:
+        # Query MAPE from forecast_snapshots where we have actuals
+        # Calculate MAPE for each model separately
+        models_to_query = ['prophet', 'xgboost', 'catboost']
+        if is_pace_metric:
+            models_to_query.append('pickup')
+
+        mape_scores = {}
+        for model in models_to_query:
+            query = text("""
+                SELECT AVG(ABS((forecast_value - actual_value) / NULLIF(actual_value, 0)) * 100) as mape
+                FROM forecast_snapshots
+                WHERE actual_value IS NOT NULL
+                    AND actual_value != 0
+                    AND forecast_value IS NOT NULL
+                    AND metric_code = :metric_code
+                    AND model = :model
+            """)
+            result = await db.execute(query, {"metric_code": snapshot_metric, "model": model})
+            row = result.fetchone()
+
+            if row and row.mape is not None:
+                mape_scores[model] = float(row.mape)
+            else:
+                mape_scores[model] = 100  # Default high MAPE if no data
+
+        # Check if we have valid MAPE data
+        if all(score == 100 for score in mape_scores.values()):
+            logger.warning(f"No MAPE data found for {snapshot_metric}, using equal weights")
+            # Fall back to equal weights
+            if is_pace_metric:
+                return {'prophet': 0.25, 'xgboost': 0.25, 'catboost': 0.25, 'pickup': 0.25}
+            else:
+                return {'prophet': 0.333, 'xgboost': 0.333, 'catboost': 0.334}
+
+        logger.info(f"MAPE scores for {snapshot_metric}: " +
+                   ", ".join([f"{k}={v:.2f}%" for k, v in mape_scores.items()]))
+
+        # Calculate inverse-MAPE weights (lower MAPE = higher weight)
+        weights = {model: 1.0 / max(mape, 0.1) for model, mape in mape_scores.items()}
+
+        # Normalize weights to sum to 1.0
+        weight_sum = sum(weights.values())
+        normalized_weights = {k: v / weight_sum for k, v in weights.items()}
+
+        logger.info(f"Normalized weights for {snapshot_metric}: " +
+                   ", ".join([f"{k}={v:.4f}" for k, v in normalized_weights.items()]))
+        return normalized_weights
+
+    except Exception as e:
+        logger.error(f"Failed to calculate model weights: {e}")
+        import traceback
+        traceback.print_exc()
+        # Fall back to equal weights
+        if is_pace_metric:
+            return {'prophet': 0.25, 'xgboost': 0.25, 'catboost': 0.25, 'pickup': 0.25}
+        else:
+            return {'prophet': 0.333, 'xgboost': 0.333, 'catboost': 0.334}
+
+
+async def run_blended_tuned_weighted_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    save_to_db: bool = False,
+    run_id: Optional[str] = None,
+    perception_date: Optional[date] = None,
+    apply_60_40_blend: bool = True
+) -> List[Dict]:
+    """
+    Generate blended forecast using production-tuned models with accuracy-based weighting.
+
+    This uses MAPE scores from the last 90 days to weight models by accuracy.
+    Lower MAPE (more accurate) models receive higher weights.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast (e.g., 'hotel_occupancy_pct', 'hotel_room_nights')
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        save_to_db: If True, saves forecasts to database (for snapshots)
+        run_id: Run ID for tracking (required if save_to_db=True)
+        perception_date: Optional date to generate forecast as-of (for backtesting)
+        apply_60_40_blend: If True, applies 60/40 blend with budget/prior year (default True)
+
+    Returns:
+        List of forecast dicts with date and predicted_value
+    """
+    logger.info(f"Running blended tuned WEIGHTED forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Check if metric is a pace metric (uses pickup)
+    is_pace_metric = metric_code in ('hotel_occupancy_pct', 'hotel_room_nights')
+
+    # Get accuracy-based weights
+    weights = await get_model_weights(db, metric_code, is_pace_metric)
+
+    # Step 1: Run individual tuned models
+    forecasts_by_date = {}
+
+    # Run Prophet Tuned
+    try:
+        from services.forecasting.prophet_tuned import run_prophet_tuned_forecast
+        prophet_forecasts = await run_prophet_tuned_forecast(db, metric_code, start_date, end_date, perception_date)
+        for fc in prophet_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['prophet'] = float(fc['predicted_value'])
+        logger.info(f"Prophet tuned generated {len(prophet_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"Prophet tuned forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run XGBoost Tuned
+    try:
+        from services.forecasting.xgboost_tuned import run_xgboost_tuned_forecast
+        xgboost_forecasts = await run_xgboost_tuned_forecast(db, metric_code, start_date, end_date, perception_date)
+        for fc in xgboost_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['xgboost'] = float(fc['predicted_value'])
+        logger.info(f"XGBoost tuned generated {len(xgboost_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"XGBoost tuned forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run CatBoost Tuned
+    try:
+        from services.forecasting.catboost_tuned import run_catboost_tuned_forecast
+        catboost_forecasts = await run_catboost_tuned_forecast(db, metric_code, start_date, end_date, perception_date)
+        for fc in catboost_forecasts:
+            fc_date = str(fc['forecast_date'])
+            if fc_date not in forecasts_by_date:
+                forecasts_by_date[fc_date] = {}
+            forecasts_by_date[fc_date]['catboost'] = float(fc['predicted_value'])
+        logger.info(f"CatBoost tuned generated {len(catboost_forecasts)} forecasts for {metric_code}")
+        db.commit()  # Commit after successful model run
+    except Exception as e:
+        logger.error(f"CatBoost tuned forecast failed for {metric_code}: {e}")
+        db.rollback()
+        db.commit()  # Start fresh transaction
+
+    # Run Pickup Tuned (only for pace metrics)
+    if is_pace_metric:
+        try:
+            from services.forecasting.pickup_tuned import run_pickup_tuned_forecast
+            pickup_forecasts = await run_pickup_tuned_forecast(db, metric_code, start_date, end_date, perception_date)
+            for fc in pickup_forecasts:
+                fc_date = str(fc['forecast_date'])
+                if fc_date not in forecasts_by_date:
+                    forecasts_by_date[fc_date] = {}
+                forecasts_by_date[fc_date]['pickup'] = float(fc['predicted_value'])
+            logger.info(f"Pickup tuned generated {len(pickup_forecasts)} forecasts for {metric_code}")
+            db.commit()  # Commit after successful model run
+        except Exception as e:
+            logger.error(f"Pickup tuned forecast failed for {metric_code}: {e}")
+            db.rollback()
+            db.commit()  # Start fresh transaction
+
+    # Step 2: Calculate MAPE-weighted model blend, then apply 60/40 with prior year/budget
+    # Determine if this is a revenue metric (uses budget instead of prior year)
+    revenue_metrics = ['net_accom', 'net_dry', 'net_wet', 'total_rev', 'hotel_arr']
+    is_revenue_metric = metric_code in revenue_metrics
+
+    blended_forecasts = []
+    for fc_date, model_forecasts in forecasts_by_date.items():
+        # Need at least 2 models to blend
+        if len(model_forecasts) < 2:
+            continue
+
+        # Apply accuracy-based weights to get weighted model blend
+        weighted_sum = 0.0
+        weight_total = 0.0
+
+        for model_name, forecast_value in model_forecasts.items():
+            if model_name in weights:
+                weighted_sum += forecast_value * weights[model_name]
+                weight_total += weights[model_name]
+
+        # Calculate weighted model average
+        if weight_total > 0:
+            weighted_model_blend = weighted_sum / weight_total
+        else:
+            # Fall back to simple average if weights are missing
+            weighted_model_blend = sum(model_forecasts.values()) / len(model_forecasts)
+
+        # Apply 60/40 blend with prior year or budget (if enabled)
+        final_value = weighted_model_blend  # Default: use model blend only
+
+        if apply_60_40_blend:
+            try:
+                forecast_date_obj = date.fromisoformat(fc_date)
+
+                if is_revenue_metric:
+                    # Revenue metrics: 60% model + 40% budget
+                    budget_query = text("""
+                        SELECT budget_value
+                        FROM daily_budgets
+                        WHERE date = :fc_date AND budget_type = :metric_code
+                    """)
+                    budget_result = await db.execute(budget_query, {
+                        "fc_date": forecast_date_obj,
+                        "metric_code": metric_code
+                    })
+                    budget_row = budget_result.fetchone()
+                    if budget_row and budget_row.budget_value is not None:
+                        budget_value = float(budget_row.budget_value)
+                        final_value = 0.6 * weighted_model_blend + 0.4 * budget_value
+                        logger.debug(f"{fc_date}: Model={weighted_model_blend:.2f}, Budget={budget_value:.2f}, Final={final_value:.2f}")
+                else:
+                    # Non-revenue metrics: 60% model + 40% prior year
+                    # Map metric codes for daily_metrics table
+                    daily_metric_map = {
+                        'hotel_occupancy_pct': 'hotel_occupancy_pct',
+                        'hotel_room_nights': 'hotel_room_nights',
+                        'hotel_guests': 'hotel_guests',
+                        'ave_guest_rate': 'ave_guest_rate',
+                    }
+                    daily_metric_code = daily_metric_map.get(metric_code, metric_code)
+
+                    # Get prior year date (same day of week, ~52 weeks back)
+                    from utils.time_alignment import get_prior_year_daily
+                    prior_year_date = get_prior_year_daily(forecast_date_obj)
+
+                    prior_query = text("""
+                        SELECT actual_value
+                        FROM daily_metrics
+                        WHERE date = :prior_date AND metric_code = :metric_code
+                    """)
+                    prior_result = await db.execute(prior_query, {
+                        "prior_date": prior_year_date,
+                        "metric_code": daily_metric_code
+                    })
+                    prior_row = prior_result.fetchone()
+                    if prior_row and prior_row.actual_value is not None:
+                        prior_value = float(prior_row.actual_value)
+                        final_value = 0.6 * weighted_model_blend + 0.4 * prior_value
+                        logger.debug(f"{fc_date}: Model={weighted_model_blend:.2f}, PriorYear={prior_value:.2f}, Final={final_value:.2f}")
+
+            except Exception as e:
+                logger.warning(f"Could not apply 60/40 blend for {fc_date}: {e}, using model blend only")
+                final_value = weighted_model_blend
+
+        blended_forecasts.append({
+            'date': fc_date,
+            'predicted_value': round(final_value, 2)
+        })
+
+    logger.info(f"Generated {len(blended_forecasts)} blended tuned WEIGHTED forecasts for {metric_code}")
+
+    # Step 3: Optionally save to database (for snapshots)
+    if save_to_db and run_id:
+        try:
+            for fc in blended_forecasts:
+                db.execute(
+                    text("""
+                        INSERT INTO forecasts
+                            (run_id, forecast_date, forecast_type, model_type, predicted_value, generated_at)
+                        VALUES
+                            (:run_id, :forecast_date, :forecast_type, 'blended_tuned_weighted', :predicted_value, NOW())
+                    """),
+                    {
+                        "run_id": run_id,
+                        "forecast_date": fc['date'],
+                        "forecast_type": metric_code,
+                        "predicted_value": fc['predicted_value']
+                    }
+                )
+            db.commit()
+            logger.info(f"Saved {len(blended_forecasts)} blended tuned weighted forecasts to database")
+        except Exception as e:
+            logger.error(f"Failed to save blended tuned weighted forecasts to database: {e}")
+            db.rollback()
+            raise
+
+    return blended_forecasts

backend/services/forecasting/budget_service.py

@@ -0,0 +1,254 @@
+"""
+Budget distribution service
+Distributes monthly budgets to daily values using prior year patterns
+"""
+import logging
+from datetime import date, timedelta
+from calendar import monthrange
+from typing import Optional, Dict, List
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+# Map budget_type to column name in newbook_net_revenue_data
+BUDGET_TYPE_TO_COLUMN = {
+    'net_accom': 'accommodation',
+    'net_dry': 'dry',
+    'net_wet': 'wet',
+}
+
+
+async def distribute_budget(
+    db,
+    year: int,
+    month: int,
+    budget_type: Optional[str] = None
+) -> dict:
+    """
+    Distribute monthly budget to daily values using DOW-aligned prior year patterns.
+
+    Uses 364-day offset (52 weeks) so that days of week align between years.
+    This ensures weekday/weekend patterns are preserved in the distribution.
+
+    Logic:
+    1. Get monthly budget from FD-provided values
+    2. For each day in target month, find DOW-aligned date 364 days prior
+    3. Get prior year actual values for those DOW-aligned dates
+    4. Calculate percentages and distribute budget accordingly
+
+    Example:
+    - Feb 1, 2026 (Sunday) - 364 days = Feb 2, 2025 (Sunday)
+    - If Feb 2, 2025 was 4% of the DOW-aligned period total
+    - Feb 1, 2026 daily budget = monthly_budget × 4%
+
+    Args:
+        db: Database session
+        year: Year to distribute for
+        month: Month to distribute (1-12)
+        budget_type: Optional specific budget type, or all if None
+
+    Returns:
+        Dict with distribution results
+    """
+    logger.info(f"Distributing budget for {year}-{month:02d} using DOW-aligned prior year patterns")
+
+    # Get monthly budgets
+    query = """
+        SELECT id, budget_type, budget_value
+        FROM monthly_budgets
+        WHERE year = :year AND month = :month
+    """
+    params = {"year": year, "month": month}
+
+    if budget_type:
+        query += " AND budget_type = :budget_type"
+        params["budget_type"] = budget_type
+
+    result = await db.execute(text(query), params)
+    monthly_budgets = result.fetchall()
+
+    if not monthly_budgets:
+        logger.warning(f"No monthly budgets found for {year}-{month:02d}")
+        return {"days_distributed": 0, "status": "no_budgets_found"}
+
+    # Get days in target month
+    _, days_in_month = monthrange(year, month)
+
+    days_distributed = 0
+
+    for budget in monthly_budgets:
+        budget_id = budget.id
+        btype = budget.budget_type
+        monthly_value = float(budget.budget_value)
+
+        # Get column name for this budget type
+        column_name = BUDGET_TYPE_TO_COLUMN.get(btype)
+        if not column_name:
+            logger.warning(f"Unknown budget type: {btype}, skipping")
+            continue
+
+        logger.info(f"Distributing {btype}: £{monthly_value:,.2f}")
+
+        # Build list of target dates and their DOW-aligned prior year dates
+        target_dates = []
+        prior_dates = []
+        for day in range(1, days_in_month + 1):
+            target_date = date(year, month, day)
+            prior_date = target_date - timedelta(days=364)  # 52 weeks back, DOW aligned
+            target_dates.append(target_date)
+            prior_dates.append(prior_date)
+
+        # Get prior year actual values for the DOW-aligned dates from newbook_net_revenue_data
+        prior_result = await db.execute(
+            text(f"""
+                SELECT date, {column_name} as value
+                FROM newbook_net_revenue_data
+                WHERE date = ANY(:dates)
+                    AND {column_name} IS NOT NULL
+                ORDER BY date
+            """),
+            {"dates": prior_dates}
+        )
+        prior_rows = prior_result.fetchall()
+        prior_values = {row.date: float(row.value) for row in prior_rows}
+
+        # Calculate total of prior year values for percentage calculation
+        total_prior = sum(prior_values.get(d, 0) for d in prior_dates)
+
+        logger.info(f"Prior year DOW-aligned total for {btype}: £{total_prior:,.2f} ({len(prior_values)} days with data)")
+
+        if total_prior > 0:
+            # Distribute using DOW-aligned prior year percentages
+            for target_date, prior_date in zip(target_dates, prior_dates):
+                prior_value = prior_values.get(prior_date, 0)
+                pct_of_total = prior_value / total_prior if total_prior > 0 else (1 / days_in_month)
+                daily_budget = monthly_value * pct_of_total
+
+                await db.execute(
+                    text("""
+                        INSERT INTO daily_budgets (
+                            date, budget_type, budget_value, distribution_method,
+                            prior_year_pct, monthly_budget_id, calculated_at
+                        ) VALUES (
+                            :date, :btype, :value, 'dow_aligned',
+                            :pct, :budget_id, NOW()
+                        )
+                        ON CONFLICT (date, budget_type) DO UPDATE SET
+                            budget_value = :value,
+                            distribution_method = 'dow_aligned',
+                            prior_year_pct = :pct,
+                            calculated_at = NOW()
+                    """),
+                    {
+                        "date": target_date,
+                        "btype": btype,
+                        "value": round(daily_budget, 2),
+                        "pct": round(pct_of_total, 6),
+                        "budget_id": budget_id
+                    }
+                )
+                days_distributed += 1
+
+            logger.info(f"Distributed {btype} using DOW-aligned prior year patterns")
+        else:
+            # No prior year data - distribute evenly
+            logger.warning(f"No prior year DOW-aligned data for {btype}, using even distribution")
+            daily_budget = monthly_value / days_in_month
+
+            for target_date in target_dates:
+                await db.execute(
+                    text("""
+                        INSERT INTO daily_budgets (
+                            date, budget_type, budget_value, distribution_method,
+                            prior_year_pct, monthly_budget_id, calculated_at
+                        ) VALUES (
+                            :date, :btype, :value, 'even',
+                            :pct, :budget_id, NOW()
+                        )
+                        ON CONFLICT (date, budget_type) DO UPDATE SET
+                            budget_value = :value,
+                            distribution_method = 'even',
+                            prior_year_pct = :pct,
+                            calculated_at = NOW()
+                    """),
+                    {
+                        "date": target_date,
+                        "btype": btype,
+                        "value": round(daily_budget, 2),
+                        "pct": round(1 / days_in_month, 6),
+                        "budget_id": budget_id
+                    }
+                )
+                days_distributed += 1
+
+    await db.commit()
+    logger.info(f"Budget distribution complete: {days_distributed} days")
+    return {"days_distributed": days_distributed, "status": "success"}
+
+
+async def calculate_prior_year_percentages(db, metric_type: str):
+    """
+    Calculate and store prior year daily percentages for budget distribution
+
+    For each month, calculates what percentage each day was of the monthly total
+    """
+    logger.info(f"Calculating prior year percentages for {metric_type}")
+
+    # Get all daily values from prior year
+    result = await db.execute(
+        text("""
+        WITH monthly_totals AS (
+            SELECT
+                EXTRACT(YEAR FROM date) as year,
+                EXTRACT(MONTH FROM date) as month,
+                SUM(actual_value) as month_total
+            FROM daily_metrics
+            WHERE metric_code = :metric_type
+                AND date >= CURRENT_DATE - INTERVAL '2 years'
+                AND actual_value IS NOT NULL
+            GROUP BY EXTRACT(YEAR FROM date), EXTRACT(MONTH FROM date)
+        )
+        SELECT
+            dm.date,
+            dm.actual_value,
+            mt.month_total,
+            dm.actual_value / NULLIF(mt.month_total, 0) as pct_of_month
+        FROM daily_metrics dm
+        JOIN monthly_totals mt ON
+            EXTRACT(YEAR FROM dm.date) = mt.year AND
+            EXTRACT(MONTH FROM dm.date) = mt.month
+        WHERE dm.metric_code = :metric_type
+            AND dm.actual_value IS NOT NULL
+        ORDER BY dm.date
+        """),
+        {"metric_type": metric_type}
+    )
+
+    count = 0
+    for row in result.fetchall():
+        await db.execute(
+            text("""
+            INSERT INTO prior_year_daily (
+                date, metric_type, actual_value, month_total, pct_of_month, fetched_at
+            ) VALUES (
+                :date, :metric, :actual, :month_total, :pct, NOW()
+            )
+            ON CONFLICT (date, metric_type) DO UPDATE SET
+                actual_value = :actual,
+                month_total = :month_total,
+                pct_of_month = :pct,
+                fetched_at = NOW()
+            """),
+            {
+                "date": row.date,
+                "metric": metric_type,
+                "actual": row.actual_value,
+                "month_total": row.month_total,
+                "pct": row.pct_of_month
+            }
+        )
+        count += 1
+
+    await db.commit()
+    logger.info(f"Prior year percentages calculated: {count} records for {metric_type}")
+    return count

backend/services/forecasting/catboost_model.py

@@ -0,0 +1,450 @@
+"""
+CatBoost forecasting model
+Gradient boosting with native categorical feature support and better out-of-box performance.
+Similar to XGBoost but handles categorical features natively without encoding.
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+import pandas as pd
+import numpy as np
+import json
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+def create_features(df: pd.DataFrame, special_dates: set = None) -> pd.DataFrame:
+    """
+    Create features for CatBoost model.
+
+    CatBoost handles categorical features natively, so we keep day_of_week as categorical
+    instead of using cyclical encoding.
+    """
+    df = df.copy()
+
+    # Date features - keep as categorical for CatBoost
+    df['day_of_week'] = df['ds'].dt.dayofweek.astype(str)  # Categorical
+    df['month'] = df['ds'].dt.month.astype(str)  # Categorical
+    df['day_of_month'] = df['ds'].dt.day
+    df['week_of_year'] = df['ds'].dt.isocalendar().week.astype(int)
+    df['is_weekend'] = (df['ds'].dt.dayofweek >= 5).astype(int)
+
+    # Special dates / holidays
+    if special_dates and len(special_dates) > 0:
+        df['is_holiday'] = df['ds'].dt.date.apply(lambda x: 1 if x in special_dates else 0)
+
+        # Days to nearest special date
+        def days_to_nearest(d):
+            if not special_dates:
+                return 30
+            future = [s for s in special_dates if s >= d]
+            if not future:
+                return 30
+            return min((s - d).days for s in future)
+
+        df['days_to_holiday'] = df['ds'].dt.date.apply(days_to_nearest)
+    else:
+        df['is_holiday'] = 0
+        df['days_to_holiday'] = 30
+
+    # Lag features
+    for lag in [7, 14, 21, 28]:
+        df[f'lag_{lag}'] = df['y'].shift(lag)
+
+    # Rolling averages
+    for window in [7, 14, 28]:
+        df[f'rolling_mean_{window}'] = df['y'].rolling(window=window, min_periods=1).mean()
+        df[f'rolling_std_{window}'] = df['y'].rolling(window=window, min_periods=1).std().fillna(0)
+
+    # Year-over-year feature (364 days for DOW alignment)
+    if len(df) > 364:
+        df['lag_364'] = df['y'].shift(364)
+
+    return df
+
+
+async def run_catboost_forecast(
+    db,
+    metric_code: str,
+    forecast_from: date,
+    forecast_to: date,
+    training_days: int = 2555,  # ~7 years
+    use_special_dates: bool = True,
+    use_otb_data: bool = True
+) -> List[dict]:
+    """
+    Run CatBoost forecast for a metric.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast
+        forecast_from: Start date for forecasts
+        forecast_to: End date for forecasts
+        training_days: Days of historical data to use
+        use_special_dates: Include holiday features
+        use_otb_data: Include OTB pickup features
+
+    Returns:
+        List of forecast records
+    """
+    try:
+        from catboost import CatBoostRegressor
+
+        # Get historical data
+        training_from = forecast_from - timedelta(days=training_days + 400)  # Extra for lag features
+
+        # Revenue metrics use earned_revenue_data joined with gl_accounts
+        revenue_metrics = ['net_accom', 'net_dry', 'net_wet', 'total_rev']
+        if metric_code in revenue_metrics:
+            revenue_departments = {
+                'net_accom': 'accommodation',
+                'net_dry': 'dry',
+                'net_wet': 'wet',
+                'total_rev': None,  # All departments
+            }
+            department = revenue_departments.get(metric_code)
+            if department is None and metric_code != 'total_rev':
+                logger.warning(f"Unknown revenue metric for CatBoost: {metric_code}")
+                return []
+
+            if metric_code == 'total_rev':
+                # Total revenue across all departments
+                result = db.execute(
+                    text("""
+                    SELECT date, SUM(amount_net) as actual_value
+                    FROM newbook_earned_revenue_data
+                    WHERE date BETWEEN :from_date AND :to_date
+                    GROUP BY date
+                    HAVING SUM(amount_net) IS NOT NULL
+                    ORDER BY date
+                    """),
+                    {"from_date": training_from, "to_date": forecast_from - timedelta(days=1)}
+                )
+            else:
+                # Revenue by department
+                result = db.execute(
+                    text("""
+                    SELECT r.date, SUM(r.amount_net) as actual_value
+                    FROM newbook_earned_revenue_data r
+                    JOIN newbook_gl_accounts g ON r.gl_account_id = g.gl_account_id
+                    WHERE r.date BETWEEN :from_date AND :to_date
+                        AND g.department = :department
+                    GROUP BY r.date
+                    HAVING SUM(r.amount_net) IS NOT NULL
+                    ORDER BY r.date
+                    """),
+                    {"from_date": training_from, "to_date": forecast_from - timedelta(days=1), "department": department}
+                )
+        else:
+            # Hotel metrics use newbook_bookings_stats table
+            metric_column_map = {
+                'hotel_occupancy_pct': 'total_occupancy_pct',
+                'hotel_room_nights': 'booking_count',
+                'hotel_guests': 'guests_count',
+            }
+
+            column_name = metric_column_map.get(metric_code)
+            if not column_name:
+                logger.warning(f"Unknown metric_code for CatBoost: {metric_code}")
+                return []
+
+            result = db.execute(
+                text(f"""
+                SELECT date, {column_name} as actual_value
+                FROM newbook_bookings_stats
+                WHERE date BETWEEN :from_date AND :to_date
+                    AND {column_name} IS NOT NULL
+                ORDER BY date
+                """),
+                {"from_date": training_from, "to_date": forecast_from - timedelta(days=1)}
+            )
+
+        rows = result.fetchall()
+
+        if len(rows) < 60:
+            logger.warning(f"Insufficient data for CatBoost: {metric_code} has {len(rows)} records")
+            return []
+
+        # Prepare DataFrame
+        df = pd.DataFrame([{"ds": pd.Timestamp(row.date), "y": float(row.actual_value)} for row in rows])
+        df = df.sort_values('ds').reset_index(drop=True)
+
+        # Load special dates if enabled
+        special_dates = None
+        if use_special_dates:
+            special_dates = await _load_special_dates(db, forecast_from, forecast_to)
+
+        # Create features
+        df = create_features(df, special_dates)
+
+        # Add OTB features if enabled
+        if use_otb_data:
+            otb_df = _load_otb_data(db, training_from, forecast_from)
+            if otb_df is not None and len(otb_df) > 0:
+                df = _add_otb_features(df, otb_df)
+                logger.info("Added OTB features to CatBoost training data")
+
+        # Remove rows with NaN from lag features
+        df = df.dropna(subset=['lag_7', 'lag_14', 'lag_21', 'lag_28'])
+
+        # Define feature columns
+        categorical_features = ['day_of_week', 'month']
+
+        numerical_features = [
+            'day_of_month', 'week_of_year', 'is_weekend',
+            'is_holiday', 'days_to_holiday',
+            'lag_7', 'lag_14', 'lag_21', 'lag_28',
+            'rolling_mean_7', 'rolling_mean_14', 'rolling_mean_28',
+            'rolling_std_7', 'rolling_std_14', 'rolling_std_28'
+        ]
+
+        # Add lag_364 if available
+        if 'lag_364' in df.columns and df['lag_364'].notna().sum() > 30:
+            numerical_features.append('lag_364')
+
+        # Add OTB features if present
+        otb_cols = ['otb_at_30d', 'otb_at_14d', 'otb_at_7d',
+                    'pickup_30d_to_14d', 'pickup_14d_to_7d',
+                    'otb_pct_at_30d', 'otb_pct_at_14d', 'otb_pct_at_7d']
+        for col in otb_cols:
+            if col in df.columns:
+                numerical_features.append(col)
+
+        feature_cols = categorical_features + numerical_features
+
+        X = df[feature_cols].copy()
+        y = df['y']
+
+        # Train CatBoost model
+        model = CatBoostRegressor(
+            iterations=200,
+            depth=6,
+            learning_rate=0.1,
+            loss_function='RMSE',
+            cat_features=categorical_features,
+            verbose=False,
+            random_seed=42
+        )
+        model.fit(X, y)
+
+        # Generate forecasts
+        forecasts = []
+        current_df = df.copy()
+
+        for forecast_date in pd.date_range(start=forecast_from, end=forecast_to, freq='D'):
+            # Create row for forecast date
+            new_row = pd.DataFrame([{"ds": forecast_date, "y": np.nan}])
+            current_df = pd.concat([current_df, new_row], ignore_index=True)
+            current_df = create_features(current_df, special_dates)
+
+            # Add OTB features for future dates if available
+            if use_otb_data:
+                current_df = _add_otb_features(current_df, otb_df)
+
+            # Get features for prediction
+            X_pred = current_df[feature_cols].iloc[-1:].copy()
+
+            # Forward fill any NaN values
+            for col in numerical_features:
+                if col in X_pred.columns:
+                    X_pred[col] = X_pred[col].ffill()
+                    if X_pred[col].isna().any():
+                        X_pred[col] = X_pred[col].fillna(0)
+
+            # Make prediction
+            prediction = float(model.predict(X_pred)[0])
+
+            # Ensure non-negative
+            prediction = max(0, prediction)
+
+            # Update y value for lag features
+            current_df.iloc[-1, current_df.columns.get_loc('y')] = prediction
+
+            forecast_record = {
+                "forecast_date": forecast_date.date(),
+                "forecast_type": metric_code,
+                "model_type": "catboost",
+                "predicted_value": round(float(prediction), 2)
+            }
+            forecasts.append(forecast_record)
+
+            # Store in database
+            db.execute(
+                text("""
+                INSERT INTO forecasts (
+                    forecast_date, forecast_type, model_type, predicted_value, generated_at
+                ) VALUES (
+                    :forecast_date, :forecast_type, :model_type, :predicted_value, NOW()
+                )
+                """),
+                forecast_record
+            )
+
+        db.commit()
+
+        # Calculate feature importance for explainability
+        try:
+            feature_importance = dict(zip(feature_cols, model.feature_importances_.tolist()))
+            top_features = sorted(
+                [{"feature": k, "importance": v} for k, v in feature_importance.items()],
+                key=lambda x: x["importance"], reverse=True
+            )[:10]
+
+            logger.info(f"CatBoost top features: {[f['feature'] for f in top_features[:5]]}")
+        except Exception as e:
+            logger.warning(f"Feature importance calculation failed: {e}")
+
+        logger.info(f"CatBoost forecast generated for {metric_code}: {len(forecasts)} records")
+        return forecasts
+
+    except ImportError as e:
+        logger.error(f"CatBoost not installed: {e}")
+        return []
+    except Exception as e:
+        logger.error(f"CatBoost forecast failed for {metric_code}: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
+        return []
+
+
+async def _load_special_dates(db, from_date: date, to_date: date) -> set:
+    """Load special dates from system_config."""
+    try:
+        result = db.execute(text("""
+            SELECT config_value FROM system_config
+            WHERE config_key = 'special_dates'
+        """))
+        row = result.fetchone()
+
+        if not row or not row.config_value:
+            return set()
+
+        dates_json = json.loads(row.config_value)
+        special_dates = set()
+
+        for item in dates_json:
+            if isinstance(item, dict) and 'date' in item:
+                try:
+                    d = pd.to_datetime(item['date']).date()
+                    special_dates.add(d)
+                except:
+                    pass
+
+        logger.info(f"Loaded {len(special_dates)} special dates for CatBoost")
+        return special_dates
+
+    except Exception as e:
+        logger.warning(f"Failed to load special dates: {e}")
+        return set()
+
+
+def _load_otb_data(db, from_date: date, to_date: date) -> Optional[pd.DataFrame]:
+    """Load OTB (On-The-Books) data."""
+    try:
+        result = db.execute(text("""
+            SELECT
+                arrival_date,
+                d93 as otb_at_90d,
+                d65 as otb_at_60d,
+                d30 as otb_at_30d,
+                d14 as otb_at_14d,
+                d7 as otb_at_7d,
+                d0 as final_bookings
+            FROM newbook_booking_pace
+            WHERE arrival_date BETWEEN :from_date AND :to_date
+            ORDER BY arrival_date
+        """), {"from_date": from_date, "to_date": to_date})
+        rows = result.fetchall()
+
+        if not rows:
+            return None
+
+        df = pd.DataFrame([{
+            "arrival_date": row.arrival_date,
+            "otb_at_90d": float(row.otb_at_90d) if row.otb_at_90d else 0,
+            "otb_at_60d": float(row.otb_at_60d) if row.otb_at_60d else 0,
+            "otb_at_30d": float(row.otb_at_30d) if row.otb_at_30d else 0,
+            "otb_at_14d": float(row.otb_at_14d) if row.otb_at_14d else 0,
+            "otb_at_7d": float(row.otb_at_7d) if row.otb_at_7d else 0,
+            "final_bookings": float(row.final_bookings) if row.final_bookings else 0
+        } for row in rows])
+
+        return df
+
+    except Exception as e:
+        logger.warning(f"Failed to load OTB data: {e}")
+        return None
+
+
+def _add_otb_features(df: pd.DataFrame, otb_df: Optional[pd.DataFrame]) -> pd.DataFrame:
+    """Add OTB features to DataFrame."""
+    df = df.copy()
+
+    if otb_df is None or len(otb_df) == 0:
+        # No OTB data - set defaults
+        df['otb_at_30d'] = 0
+        df['otb_at_14d'] = 0
+        df['otb_at_7d'] = 0
+        df['pickup_30d_to_14d'] = 0
+        df['pickup_14d_to_7d'] = 0
+        df['otb_pct_at_30d'] = 0
+        df['otb_pct_at_14d'] = 0
+        df['otb_pct_at_7d'] = 0
+        return df
+
+    # Create date column for merging
+    df['date_only'] = df['ds'].dt.date
+
+    # Merge OTB data
+    otb_df = otb_df.copy()
+    otb_df['date_only'] = pd.to_datetime(otb_df['arrival_date']).dt.date
+
+    # Check if columns already exist (avoid duplicates)
+    merge_cols = ['date_only']
+    for col in ['otb_at_90d', 'otb_at_60d', 'otb_at_30d', 'otb_at_14d', 'otb_at_7d', 'final_bookings']:
+        if col not in df.columns:
+            merge_cols.append(col)
+
+    if len(merge_cols) > 1:
+        df = df.merge(
+            otb_df[merge_cols],
+            on='date_only',
+            how='left'
+        )
+
+    # Fill NaN with 0
+    for col in ['otb_at_90d', 'otb_at_60d', 'otb_at_30d', 'otb_at_14d', 'otb_at_7d', 'final_bookings']:
+        if col in df.columns:
+            df[col] = df[col].fillna(0)
+
+    # Calculate pickup between windows
+    if 'pickup_30d_to_14d' not in df.columns:
+        df['pickup_30d_to_14d'] = df['otb_at_14d'] - df['otb_at_30d']
+    if 'pickup_14d_to_7d' not in df.columns:
+        df['pickup_14d_to_7d'] = df['otb_at_7d'] - df['otb_at_14d']
+
+    # Calculate OTB as percentage of final (capped at 100%)
+    if 'otb_pct_at_30d' not in df.columns:
+        df['otb_pct_at_30d'] = np.where(
+            df['final_bookings'] > 0,
+            np.minimum(df['otb_at_30d'] / df['final_bookings'] * 100, 100),
+            0
+        )
+    if 'otb_pct_at_14d' not in df.columns:
+        df['otb_pct_at_14d'] = np.where(
+            df['final_bookings'] > 0,
+            np.minimum(df['otb_at_14d'] / df['final_bookings'] * 100, 100),
+            0
+        )
+    if 'otb_pct_at_7d' not in df.columns:
+        df['otb_pct_at_7d'] = np.where(
+            df['final_bookings'] > 0,
+            np.minimum(df['otb_at_7d'] / df['final_bookings'] * 100, 100),
+            0
+        )
+
+    # Drop temporary column
+    df = df.drop(columns=['date_only'], errors='ignore')
+
+    return df

backend/services/forecasting/catboost_tuned.py

@@ -0,0 +1,449 @@
+"""
+CatBoost Tuned Model Service
+
+This is the production-tuned CatBoost model extracted from the catboost-preview endpoint.
+Uses the exact same logic as the frontend preview to ensure value consistency.
+
+Features:
+- 2 years of training data
+- Native categorical feature support (day_of_week, month)
+- Pace features (OTB at different lead times) for room-based metrics
+- Time-based features
+- Lag features from prior year
+- OTB floor capping
+- Per-date bookable cap adjustments
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Dict, Optional
+import pandas as pd
+import numpy as np
+from catboost import CatBoostRegressor
+import warnings
+from sqlalchemy import text
+
+from utils.capacity import get_bookable_cap
+from api.special_dates import resolve_special_date
+
+logger = logging.getLogger(__name__)
+warnings.filterwarnings('ignore')
+
+
+# Metric configuration mapping
+METRIC_COLUMN_MAP = {
+    'occupancy': ('s.occupancy_pct', False, True),
+    'rooms': ('s.booking_count', False, False),
+    'guests': ('s.guest_count', False, False),
+    'ave_guest_rate': ('s.arr_net', False, False),
+    'arr': ('s.arr_net', False, False),
+    'net_accom': ('r.accommodation', True, False),
+    'net_dry': ('r.dry', True, False),
+    'net_wet': ('r.wet', True, False),
+    'total_rev': ('(COALESCE(r.accommodation, 0) + COALESCE(r.dry, 0) + COALESCE(r.wet, 0))', True, False),
+}
+
+
+def get_metric_query_parts(metric: str) -> tuple:
+    """Get SQL query parts for a metric. Returns: (column_expr, from_clause, is_percentage)"""
+    if metric not in METRIC_COLUMN_MAP:
+        metric = 'rooms'
+    col_expr, needs_revenue, is_pct = METRIC_COLUMN_MAP[metric]
+    if needs_revenue:
+        from_clause = """
+            FROM newbook_bookings_stats s
+            LEFT JOIN newbook_net_revenue_data r ON s.date = r.date
+        """
+    else:
+        from_clause = "FROM newbook_bookings_stats s"
+    return col_expr, from_clause, is_pct
+
+
+def get_lead_time_column(lead_days: int) -> str:
+    """Map lead days to the appropriate column in newbook_booking_pace."""
+    if lead_days <= 0:
+        return "d0"
+    elif lead_days <= 30:
+        return f"d{lead_days}"
+    elif lead_days <= 177:
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        for col in weekly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d177"
+    else:
+        monthly_cols = [210, 240, 270, 300, 330, 365]
+        for col in monthly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d365"
+
+
+def round_towards_reference(value: float, reference: Optional[float]) -> int:
+    """Round a forecast value towards a reference value (prior year actual)."""
+    if reference is None:
+        return round(value)
+    if value < reference:
+        return int(np.ceil(value))
+    else:
+        return int(np.floor(value))
+
+
+async def run_catboost_tuned_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    perception_date: Optional[date] = None
+) -> List[Dict]:
+    """
+    Generate CatBoost forecast using production-tuned model.
+
+    This uses the exact same logic as the catboost-preview endpoint to ensure
+    backend snapshots match frontend preview values.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        perception_date: Optional date to generate forecast as-of (for backtesting)
+
+    Returns:
+        List of forecast dicts with forecast_date and predicted_value
+    """
+    logger.info(f"Running CatBoost tuned forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Map metric codes to preview endpoint metric names
+    metric_map = {
+        'hotel_occupancy_pct': 'occupancy',
+        'hotel_room_nights': 'rooms',
+        'hotel_guests': 'guests',
+        'hotel_arr': 'arr',
+        'ave_guest_rate': 'ave_guest_rate',
+        'net_accom': 'net_accom',
+        'net_dry': 'net_dry',
+        'net_wet': 'net_wet',
+        'total_rev': 'total_rev',
+    }
+
+    metric = metric_map.get(metric_code, 'rooms')
+
+    # Use perception_date if provided, otherwise use actual today
+    today = perception_date if perception_date else date.today()
+
+    # Get default bookable cap
+    default_bookable_cap = await get_bookable_cap(db)
+
+    # Get metric column and query parts
+    col_expr, from_clause, is_pct_metric = get_metric_query_parts(metric)
+    is_room_based = metric in ('occupancy', 'rooms')
+
+    # Get historical data (2+ years for YoY features)
+    history_start = today - timedelta(days=730)
+
+    # Lead times to train on (only used for room-based metrics)
+    train_lead_times = [0, 1, 3, 7, 14, 21, 28, 30]
+
+    # Get final values (and pace data for room-based metrics)
+    if is_room_based:
+        history_result = await db.execute(text("""
+            SELECT s.date as ds, s.booking_count as final,
+                   p.d0, p.d1, p.d3, p.d7, p.d14, p.d21, p.d28, p.d30
+            FROM newbook_bookings_stats s
+            LEFT JOIN newbook_booking_pace p ON s.date = p.arrival_date
+            WHERE s.date >= :history_start
+            AND s.date < :today
+            AND s.booking_count IS NOT NULL
+            ORDER BY s.date
+        """), {"history_start": history_start, "today": today})
+    else:
+        # Non-room metrics: get values without pace join
+        history_query = f"""
+            SELECT s.date as ds, {col_expr} as final
+            {from_clause}
+            WHERE s.date >= :history_start
+            AND s.date < :today
+            AND {col_expr} IS NOT NULL
+            ORDER BY s.date
+        """
+        history_result = await db.execute(text(history_query), {"history_start": history_start, "today": today})
+
+    history_rows = history_result.fetchall()
+
+    if len(history_rows) < 30:
+        logger.warning(f"Insufficient historical data for CatBoost model: {len(history_rows)} rows")
+        return []
+
+    # Load special dates for feature
+    special_date_set = set()
+    try:
+        special_dates_result = await db.execute(text(
+            "SELECT * FROM special_dates WHERE is_active = TRUE"
+        ))
+        special_dates_rows = special_dates_result.fetchall()
+        years_needed = set(r.ds.year for r in history_rows) | {today.year, today.year + 1}
+        for row in special_dates_rows:
+            sd = {
+                'pattern_type': row.pattern_type,
+                'fixed_month': row.fixed_month,
+                'fixed_day': row.fixed_day,
+                'nth_week': row.nth_week,
+                'weekday': row.weekday,
+                'month': row.month,
+                'relative_to_month': row.relative_to_month,
+                'relative_to_day': row.relative_to_day,
+                'relative_weekday': row.relative_weekday,
+                'relative_direction': row.relative_direction,
+                'duration_days': row.duration_days,
+                'is_recurring': row.is_recurring,
+                'one_off_year': row.one_off_year
+            }
+            for year in years_needed:
+                resolved_dates = resolve_special_date(sd, year)
+                for d in resolved_dates:
+                    special_date_set.add(d)
+    except Exception as e:
+        logger.warning(f"Could not load special dates: {e}")
+
+    # Build lookup dicts
+    final_by_date = {}
+    pace_by_date = {}
+    for row in history_rows:
+        final_by_date[row.ds] = row.final
+        if is_room_based and hasattr(row, 'd0'):
+            pace_by_date[row.ds] = {
+                0: row.d0, 1: row.d1, 3: row.d3, 7: row.d7,
+                14: row.d14, 21: row.d21, 28: row.d28, 30: row.d30
+            }
+
+    # Build training examples
+    training_rows = []
+
+    if is_room_based:
+        # Room-based metrics: use pace features (one per date,lead_time combo)
+        for row in history_rows:
+            ds = row.ds
+            final = float(row.final) if row.final else 0
+            prior_ds = ds - timedelta(days=364)
+
+            prior_final = final_by_date.get(prior_ds)
+            if prior_final is None:
+                continue
+
+            for lead_time in train_lead_times:
+                current_otb = pace_by_date.get(ds, {}).get(lead_time)
+                if current_otb is None:
+                    continue
+
+                prior_otb = pace_by_date.get(prior_ds, {}).get(lead_time)
+                if prior_otb is None:
+                    prior_otb = 0
+
+                otb_pct_of_prior_final = (float(current_otb) / float(prior_final) * 100) if prior_final > 0 else 0
+
+                training_rows.append({
+                    'ds': ds,
+                    'y': final,
+                    'days_out': lead_time,
+                    'current_otb': float(current_otb),
+                    'prior_otb_same_lead': float(prior_otb),
+                    'lag_364': float(prior_final),
+                    'otb_pct_of_prior_final': otb_pct_of_prior_final
+                })
+    else:
+        # Non-room metrics: use time features only (one per date)
+        for row in history_rows:
+            ds = row.ds
+            final = float(row.final) if row.final else 0
+            prior_ds = ds - timedelta(days=364)
+
+            prior_final = final_by_date.get(prior_ds)
+            if prior_final is None:
+                prior_final = 0  # Allow training even without prior year for revenue metrics
+
+            training_rows.append({
+                'ds': ds,
+                'y': final,
+                'lag_364': float(prior_final) if prior_final else 0
+            })
+
+    if len(training_rows) < 30:
+        logger.warning(f"Insufficient data for CatBoost training: {len(training_rows)} rows")
+        return []
+
+    df = pd.DataFrame(training_rows)
+    df['ds'] = pd.to_datetime(df['ds'])
+
+    # Convert to occupancy if needed
+    if metric == "occupancy" and default_bookable_cap > 0:
+        df["y"] = (df["y"] / default_bookable_cap) * 100
+        if "current_otb" in df.columns:
+            df["current_otb"] = (df["current_otb"] / default_bookable_cap) * 100
+        if "prior_otb_same_lead" in df.columns:
+            df["prior_otb_same_lead"] = (df["prior_otb_same_lead"] / default_bookable_cap) * 100
+        df["lag_364"] = (df["lag_364"] / default_bookable_cap) * 100
+
+    # Create features - CatBoost handles categoricals natively
+    df['day_of_week'] = df['ds'].dt.dayofweek.astype(str)  # Categorical
+    df['month'] = df['ds'].dt.month.astype(str)  # Categorical
+    df['week_of_year'] = df['ds'].dt.isocalendar().week.astype(int)
+    df['is_weekend'] = (df['ds'].dt.dayofweek >= 5).astype(int)
+    df['is_special_date'] = df['ds'].dt.date.apply(lambda x: 1 if x in special_date_set else 0)
+
+    df_train = df.dropna()
+
+    if len(df_train) < 30:
+        logger.warning(f"Insufficient data after creating features: {len(df_train)} rows")
+        return []
+
+    # Define features based on metric type - categoricals handled natively by CatBoost
+    categorical_features = ['day_of_week', 'month']
+    if is_room_based:
+        numerical_features = ['week_of_year', 'is_weekend', 'is_special_date',
+                             'days_out', 'current_otb', 'prior_otb_same_lead', 'lag_364', 'otb_pct_of_prior_final']
+    else:
+        numerical_features = ['week_of_year', 'is_weekend', 'is_special_date', 'lag_364']
+    feature_cols = categorical_features + numerical_features
+
+    X_train = df_train[feature_cols]
+    y_train = df_train['y']
+
+    # Train CatBoost model
+    model = CatBoostRegressor(
+        iterations=150,
+        depth=6,
+        learning_rate=0.1,
+        loss_function='RMSE',
+        cat_features=categorical_features,
+        verbose=False,
+        random_seed=42
+    )
+    model.fit(X_train, y_train)
+
+    # Create future dataframe for forecast period
+    future_dates = []
+    current_date = start_date
+    while current_date <= end_date:
+        if (current_date - today).days >= 0:
+            future_dates.append(current_date)
+        current_date += timedelta(days=1)
+
+    if not future_dates:
+        logger.warning("No future dates to forecast")
+        return []
+
+    # Generate forecasts for each date
+    forecasts = []
+
+    for forecast_date in future_dates:
+        lead_days = (forecast_date - today).days
+        lead_col = get_lead_time_column(lead_days)
+        prior_year_date = forecast_date - timedelta(days=364)
+
+        # Get OTB data only for room-based metrics
+        current_otb = None
+
+        if is_room_based:
+            current_query = text("""
+                SELECT booking_count as current_otb
+                FROM newbook_bookings_stats
+                WHERE date = :arrival_date
+            """)
+            current_result = await db.execute(current_query, {"arrival_date": forecast_date})
+            current_row = current_result.fetchone()
+            current_otb = current_row.current_otb if current_row and current_row.current_otb is not None else 0
+
+        # Get prior year final using metric mapping
+        prior_query = f"""
+            SELECT {col_expr} as prior_final
+            {from_clause}
+            WHERE s.date = :prior_date
+        """
+        prior_result = await db.execute(text(prior_query), {"prior_date": prior_year_date})
+        prior_row = prior_result.fetchone()
+        prior_final = float(prior_row.prior_final) if prior_row and prior_row.prior_final is not None else 0
+
+        # Get per-date bookable cap
+        date_bookable_cap = await get_bookable_cap(db, forecast_date, default_bookable_cap)
+
+        forecast_dt = pd.Timestamp(forecast_date)
+        lag_364_val = prior_final if prior_final else 0
+
+        # Convert to occupancy if needed
+        if metric == "occupancy" and date_bookable_cap > 0:
+            if current_otb is not None:
+                current_otb = (current_otb / date_bookable_cap) * 100
+            lag_364_val = (prior_final / date_bookable_cap) * 100 if prior_final else 0
+
+        # Build features based on metric type
+        if is_room_based:
+            # Get prior OTB at same lead time
+            prior_year_for_otb = forecast_date - timedelta(days=364)
+            prior_otb_query = text(f"""
+                SELECT {lead_col} as prior_otb
+                FROM newbook_booking_pace
+                WHERE arrival_date = :prior_date
+            """)
+            prior_otb_result = await db.execute(prior_otb_query, {"prior_date": prior_year_for_otb})
+            prior_otb_row = prior_otb_result.fetchone()
+            prior_otb_same_lead = prior_otb_row.prior_otb if prior_otb_row and prior_otb_row.prior_otb is not None else 0
+
+            if metric == "occupancy" and date_bookable_cap > 0:
+                prior_otb_same_lead = (prior_otb_same_lead / date_bookable_cap) * 100 if prior_otb_same_lead else 0
+
+            current_otb_val = current_otb if current_otb is not None else 0
+            otb_pct_of_prior_final = (current_otb_val / lag_364_val * 100) if lag_364_val > 0 else 0
+
+            features = pd.DataFrame([{
+                'day_of_week': str(forecast_dt.dayofweek),  # Categorical
+                'month': str(forecast_dt.month),  # Categorical
+                'week_of_year': forecast_dt.isocalendar().week,
+                'is_weekend': 1 if forecast_dt.dayofweek >= 5 else 0,
+                'is_special_date': 1 if forecast_date in special_date_set else 0,
+                'days_out': lead_days,
+                'current_otb': current_otb_val,
+                'prior_otb_same_lead': prior_otb_same_lead,
+                'lag_364': lag_364_val,
+                'otb_pct_of_prior_final': otb_pct_of_prior_final,
+            }])
+        else:
+            features = pd.DataFrame([{
+                'day_of_week': str(forecast_dt.dayofweek),  # Categorical
+                'month': str(forecast_dt.month),  # Categorical
+                'week_of_year': forecast_dt.isocalendar().week,
+                'is_weekend': 1 if forecast_dt.dayofweek >= 5 else 0,
+                'is_special_date': 1 if forecast_date in special_date_set else 0,
+                'lag_364': lag_364_val,
+            }])
+
+        # Predict
+        yhat = float(model.predict(features)[0])
+
+        # Cap at max capacity based on metric type (uses per-date bookable cap)
+        if is_pct_metric:
+            yhat = min(max(yhat, 0), 100.0)
+        elif metric == 'rooms':
+            yhat = round(min(max(yhat, 0), float(date_bookable_cap)))
+        elif metric == 'guests':
+            yhat = round(max(yhat, 0))
+        else:
+            # Revenue/rate metrics: just ensure non-negative
+            yhat = max(yhat, 0)
+
+        # Floor forecast to current OTB (room-based only)
+        if is_room_based and current_otb is not None and yhat < current_otb:
+            yhat = current_otb
+
+        # Round based on metric type
+        if metric == "occupancy":
+            yhat = round(yhat, 1)
+        else:
+            yhat = round_towards_reference(yhat, prior_final)
+
+        forecasts.append({
+            'forecast_date': forecast_date,
+            'predicted_value': yhat
+        })
+
+    logger.info(f"CatBoost tuned generated {len(forecasts)} forecasts for {metric_code}")
+    return forecasts

backend/services/forecasting/covers_model.py

@@ -0,0 +1,866 @@
+"""
+Restaurant Covers Forecast Model
+
+Forecasts restaurant covers based on:
+- Breakfast: Previous night's hotel occupancy (guests expected at breakfast)
+- Lunch: OTB bookings + non-resident pickup based on lead time
+- Dinner: OTB bookings split by hotel guest/non-resident + pickup for each segment
+
+Key segments:
+- Resident (hotel guest): Based on hotel occupancy, booking patterns, DBB packages
+- Non-resident: Based on historical pickup patterns at lead time
+"""
+import logging
+import math
+from datetime import date, timedelta
+from decimal import Decimal
+from typing import Dict, List, Optional, Any
+from collections import defaultdict
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from services.forecasting.pickup_v2_model import forecast_rooms_for_date, get_prior_year_date as get_py_date
+
+logger = logging.getLogger(__name__)
+
+# Valid booking statuses for counting
+VALID_STATUSES = ('approved', 'arrived', 'seated', 'left')
+
+
+async def get_hotel_bookings_with_dinner_reservation(
+    db: AsyncSession,
+    target_date: date
+) -> Dict[str, int]:
+    """
+    Get actual count of hotel bookings that have dinner reservations for a date.
+
+    Queries resos_bookings_data to find distinct hotel_booking_numbers
+    that have dinner reservations, then compares to total hotel bookings.
+
+    Returns:
+        {
+            "rooms_with_dinner": int,  # Hotel bookings with dinner reservation
+            "total_hotel_rooms": int,   # Total hotel bookings for this date
+            "rooms_without_dinner": int # Difference
+        }
+    """
+    # Count distinct hotel bookings with dinner reservations
+    result = await db.execute(
+        text("""
+            SELECT COUNT(DISTINCT hotel_booking_number) as rooms_with_dinner
+            FROM resos_bookings_data
+            WHERE booking_date = :target_date
+            AND is_hotel_guest = true
+            AND period_type = 'dinner'
+            AND hotel_booking_number IS NOT NULL
+            AND hotel_booking_number != ''
+            AND status IN ('approved', 'arrived', 'seated', 'left')
+        """),
+        {"target_date": target_date}
+    )
+    row = result.fetchone()
+    rooms_with_dinner = row.rooms_with_dinner if row else 0
+
+    # Get total hotel bookings from stats
+    result = await db.execute(
+        text("""
+            SELECT COALESCE(booking_count, 0) as total_rooms
+            FROM newbook_bookings_stats
+            WHERE date = :target_date
+        """),
+        {"target_date": target_date}
+    )
+    row = result.fetchone()
+    total_hotel_rooms = row.total_rooms if row else 0
+
+    rooms_without_dinner = max(0, total_hotel_rooms - rooms_with_dinner)
+
+    return {
+        "rooms_with_dinner": rooms_with_dinner,
+        "total_hotel_rooms": total_hotel_rooms,
+        "rooms_without_dinner": rooms_without_dinner,
+    }
+
+
+def get_prior_year_date(target_date: date) -> date:
+    """
+    Get prior year date with 364-day offset for day-of-week alignment.
+    52 weeks = 364 days, so Monday aligns with Monday.
+    """
+    return target_date - timedelta(days=364)
+
+
+async def get_hotel_occupancy_for_date(db: AsyncSession, stay_date: date) -> Dict[str, Any]:
+    """
+    Get hotel room occupancy for a specific date from aggregated stats.
+    Returns occupied rooms, total capacity, and occupancy percentage.
+    Uses newbook_bookings_stats which is pre-aggregated with is_included filtering.
+    """
+    # Query from aggregated stats table - more reliable and already filtered
+    result = await db.execute(
+        text("""
+            SELECT
+                COALESCE(booking_count, 0) as room_count,
+                COALESCE(guests_count, 0) as guest_count,
+                COALESCE(bookable_count, 0) as total_rooms,
+                COALESCE(bookable_occupancy_pct, 0) as occupancy_pct
+            FROM newbook_bookings_stats
+            WHERE date = :stay_date
+        """),
+        {"stay_date": stay_date}
+    )
+    row = result.fetchone()
+
+    if row:
+        return {
+            "occupied_rooms": row.room_count,
+            "total_rooms": row.total_rooms,
+            "occupancy_pct": round(float(row.occupancy_pct), 1) if row.occupancy_pct else 0,
+            "guests": row.guest_count
+        }
+
+    # No stats for this date - return empty
+    return {"occupied_rooms": 0, "total_rooms": 0, "occupancy_pct": 0, "guests": 0}
+
+
+async def get_resos_covers_for_date(
+    db: AsyncSession,
+    target_date: date,
+    period_type: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    Get restaurant booking covers for a specific date from aggregated stats table.
+    Returns covers by period (breakfast, lunch, dinner, etc.)
+    """
+    # Query from aggregated stats table - more efficient and reliable
+    result = await db.execute(
+        text("""
+            SELECT
+                COALESCE(breakfast_covers, 0) as breakfast_covers,
+                COALESCE(lunch_covers, 0) as lunch_covers,
+                COALESCE(afternoon_covers, 0) as afternoon_covers,
+                COALESCE(dinner_covers, 0) as dinner_covers,
+                COALESCE(other_covers, 0) as other_covers,
+                COALESCE(total_covers, 0) as total_covers,
+                COALESCE(hotel_guest_covers, 0) as hotel_guest_covers,
+                COALESCE(non_hotel_guest_covers, 0) as non_hotel_guest_covers,
+                COALESCE(dbb_covers, 0) as dbb_covers,
+                COALESCE(total_bookings, 0) as total_bookings
+            FROM resos_bookings_stats
+            WHERE date = :target_date
+        """),
+        {"target_date": target_date}
+    )
+    row = result.fetchone()
+
+    if not row:
+        # No data for this date - return empty structure
+        return {
+            "breakfast": {"total_covers": 0, "booking_count": 0, "resident_covers": 0, "non_resident_covers": 0, "dbb_covers": 0},
+            "lunch": {"total_covers": 0, "booking_count": 0, "resident_covers": 0, "non_resident_covers": 0, "dbb_covers": 0},
+            "dinner": {"total_covers": 0, "booking_count": 0, "resident_covers": 0, "non_resident_covers": 0, "dbb_covers": 0},
+        }
+
+    # Calculate resident/non-resident split proportionally for each period
+    # (stats table has overall split but not per-period, so we estimate based on ratio)
+    total = row.total_covers or 1  # Avoid division by zero
+    hotel_ratio = row.hotel_guest_covers / total if total > 0 else 0
+    non_hotel_ratio = row.non_hotel_guest_covers / total if total > 0 else 0
+
+    covers_by_period = {
+        "breakfast": {
+            "total_covers": row.breakfast_covers,
+            "booking_count": 0,  # Not tracked per period in stats
+            "resident_covers": int(row.breakfast_covers * hotel_ratio),
+            "non_resident_covers": int(row.breakfast_covers * non_hotel_ratio),
+            "dbb_covers": 0
+        },
+        "lunch": {
+            "total_covers": row.lunch_covers,
+            "booking_count": 0,
+            "resident_covers": int(row.lunch_covers * hotel_ratio),
+            "non_resident_covers": int(row.lunch_covers * non_hotel_ratio),
+            "dbb_covers": 0
+        },
+        "dinner": {
+            "total_covers": row.dinner_covers,
+            "booking_count": 0,
+            "resident_covers": int(row.dinner_covers * hotel_ratio),
+            "non_resident_covers": int(row.dinner_covers * non_hotel_ratio),
+            "dbb_covers": row.dbb_covers
+        },
+    }
+
+    return covers_by_period
+
+
+async def get_historical_breakfast_rate(db: AsyncSession, lookback_days: int = 90) -> float:
+    """
+    Calculate historical breakfast attendance rate as covers per occupied room.
+    Uses past data to determine typical breakfast covers per hotel room.
+    Uses aggregated stats tables for reliability.
+    """
+    # Join resos stats with newbook stats to get breakfast covers and occupancy
+    result = await db.execute(
+        text("""
+            SELECT
+                SUM(rbs.breakfast_covers) as total_breakfast,
+                SUM(nbs.booking_count) as total_room_nights
+            FROM resos_bookings_stats rbs
+            JOIN newbook_bookings_stats nbs ON rbs.date = nbs.date
+            WHERE rbs.date >= CURRENT_DATE - CAST(:lookback_days AS INTEGER)
+            AND rbs.date < CURRENT_DATE
+            AND rbs.breakfast_covers > 0
+            AND nbs.booking_count > 0
+        """),
+        {"lookback_days": lookback_days}
+    )
+    row = result.fetchone()
+
+    if row and row.total_room_nights and row.total_room_nights > 0:
+        # Calculate covers per room night
+        rate = float(row.total_breakfast) / float(row.total_room_nights)
+        return rate
+
+    # Default: assume 1.8 covers per room (average party size for breakfast)
+    return 1.8
+
+
+async def get_lunch_pickup_by_lead_time(
+    db: AsyncSession,
+    target_date: date,
+    lead_days: int,
+    lookback_weeks: int = 8
+) -> int:
+    """
+    Get the median pickup COUNT for lunch at a given lead time for the same DOW.
+
+    Pickup = final_covers - otb_at_lead
+    This tells us how many covers typically come in AFTER this lead time.
+
+    More stable than ratio-based approach because it doesn't inflate
+    when current OTB is higher than historical OTB.
+
+    Args:
+        db: Database session
+        target_date: Date we're forecasting (to get DOW)
+        lead_days: Days until the target date
+        lookback_weeks: Weeks of history to use
+
+    Returns:
+        Median pickup count (integer), or 0 if no data
+    """
+    # Get day of week - convert Python (0=Mon) to PostgreSQL (0=Sun, 1=Mon...6=Sat)
+    python_dow = target_date.weekday()
+    pg_dow = (python_dow + 1) % 7
+
+    # Determine which pace column to use based on lead days
+    if lead_days <= 0:
+        return 0  # No pickup for past dates
+    elif lead_days <= 30:
+        pace_col = f"d{lead_days}"
+    elif lead_days <= 177:
+        # Weekly intervals - find closest
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        pace_col = f"d{min(weekly_cols, key=lambda x: abs(x - lead_days))}"
+    else:
+        pace_col = "d177"  # Cap at max tracked
+
+    # Query pace data for same DOW to calculate pickup counts
+    # pace_type 'total' gives us overall covers
+    result = await db.execute(
+        text(f"""
+            SELECT
+                COALESCE({pace_col}, 0) as otb_at_lead,
+                COALESCE(d0, 0) as final_covers
+            FROM resos_booking_pace
+            WHERE EXTRACT(DOW FROM booking_date) = :dow
+            AND booking_date >= CURRENT_DATE - CAST(:lookback_days AS INTEGER)
+            AND booking_date < CURRENT_DATE
+            AND d0 > 0
+            AND pace_type = 'total'
+            ORDER BY booking_date DESC
+            LIMIT :max_weeks
+        """),
+        {
+            "dow": pg_dow,
+            "lookback_days": lookback_weeks * 7,
+            "max_weeks": lookback_weeks
+        }
+    )
+    rows = result.fetchall()
+
+    if not rows:
+        # No pace data - return 0 (no pickup estimate available)
+        return 0
+
+    # Calculate pickup counts for each historical day
+    pickups = []
+    for row in rows:
+        otb_at_lead = row.otb_at_lead or 0
+        final = row.final_covers or 0
+        # Pickup = how many came in after this lead time
+        pickup = max(0, final - otb_at_lead)  # Floor at 0 (cancellations shouldn't give negative)
+        pickups.append(pickup)
+
+    if not pickups:
+        return 0
+
+    # Calculate median pickup count
+    pickups_sorted = sorted(pickups)
+    n = len(pickups_sorted)
+    if n % 2 == 0:
+        median = (pickups_sorted[n // 2 - 1] + pickups_sorted[n // 2]) / 2
+    else:
+        median = pickups_sorted[n // 2]
+
+    return math.ceil(median)  # Round up
+
+
+async def get_dinner_non_resident_pickup_by_lead_time(
+    db: AsyncSession,
+    target_date: date,
+    lead_days: int,
+    lookback_weeks: int = 8
+) -> int:
+    """
+    Get median pickup count for non-resident dinner at a given lead time.
+    Same logic as lunch - straight pickup count based on historical pace data.
+    """
+    python_dow = target_date.weekday()
+    pg_dow = (python_dow + 1) % 7
+
+    if lead_days <= 0:
+        return 0
+    elif lead_days <= 30:
+        pace_col = f"d{lead_days}"
+    elif lead_days <= 177:
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        pace_col = f"d{min(weekly_cols, key=lambda x: abs(x - lead_days))}"
+    else:
+        pace_col = "d177"
+
+    # Query pace data for non_resident type
+    result = await db.execute(
+        text(f"""
+            SELECT
+                COALESCE({pace_col}, 0) as otb_at_lead,
+                COALESCE(d0, 0) as final_covers
+            FROM resos_booking_pace
+            WHERE EXTRACT(DOW FROM booking_date) = :dow
+            AND booking_date >= CURRENT_DATE - CAST(:lookback_days AS INTEGER)
+            AND booking_date < CURRENT_DATE
+            AND d0 > 0
+            AND pace_type = 'non_resident'
+            ORDER BY booking_date DESC
+            LIMIT :max_weeks
+        """),
+        {
+            "dow": pg_dow,
+            "lookback_days": lookback_weeks * 7,
+            "max_weeks": lookback_weeks
+        }
+    )
+    rows = result.fetchall()
+
+    if not rows:
+        return 0
+
+    pickups = []
+    for row in rows:
+        otb_at_lead = row.otb_at_lead or 0
+        final = row.final_covers or 0
+        pickup = max(0, final - otb_at_lead)
+        pickups.append(pickup)
+
+    if not pickups:
+        return 0
+
+    pickups_sorted = sorted(pickups)
+    n = len(pickups_sorted)
+    if n % 2 == 0:
+        median = (pickups_sorted[n // 2 - 1] + pickups_sorted[n // 2]) / 2
+    else:
+        median = pickups_sorted[n // 2]
+
+    return math.ceil(median)
+
+
+async def get_resident_dining_rate(
+    db: AsyncSession,
+    target_date: date,
+    lookback_weeks: int = 4
+) -> float:
+    """
+    Calculate what % of hotel guests typically dine at the restaurant (resident covers).
+
+    Simple approach: resident_covers / hotel_guests for same DOW over last N weeks.
+    Returns median rate to apply to forecasted hotel guests.
+
+    Args:
+        db: Database session
+        target_date: Date we're forecasting (to get DOW)
+        lookback_weeks: Weeks of history to analyze
+
+    Returns:
+        Median dining rate (0.0 to 1.0)
+    """
+    python_dow = target_date.weekday()
+    pg_dow = (python_dow + 1) % 7
+
+    # Query resident covers and hotel guests for same DOW
+    result = await db.execute(
+        text("""
+            SELECT
+                nbs.date,
+                COALESCE(nbs.guests_count, 0) as hotel_guests,
+                COALESCE(rbs.hotel_guest_covers, 0) as resident_covers
+            FROM newbook_bookings_stats nbs
+            JOIN resos_bookings_stats rbs ON nbs.date = rbs.date
+            WHERE EXTRACT(DOW FROM nbs.date) = :dow
+            AND nbs.date >= CURRENT_DATE - CAST(:lookback_days AS INTEGER)
+            AND nbs.date < CURRENT_DATE
+            AND nbs.guests_count > 0
+            ORDER BY nbs.date DESC
+            LIMIT :max_weeks
+        """),
+        {
+            "dow": pg_dow,
+            "lookback_days": lookback_weeks * 7,
+            "max_weeks": lookback_weeks
+        }
+    )
+    rows = result.fetchall()
+
+    if not rows:
+        return 0.4  # Default 40% if no data
+
+    # Calculate dining rate for each week
+    dining_rates = []
+    for row in rows:
+        if row.hotel_guests > 0:
+            rate = min(1.0, row.resident_covers / row.hotel_guests)
+            dining_rates.append(rate)
+
+    if not dining_rates:
+        return 0.4
+
+    # Return median
+    sorted_rates = sorted(dining_rates)
+    n = len(sorted_rates)
+    if n % 2 == 0:
+        return (sorted_rates[n // 2 - 1] + sorted_rates[n // 2]) / 2
+    return sorted_rates[n // 2]
+
+
+async def get_historical_pickup_by_lead_time(
+    db: AsyncSession,
+    period_type: str,
+    is_resident: bool,
+    lead_days: int,
+    lookback_weeks: int = 12
+) -> Dict[str, float]:
+    """
+    Calculate historical pickup patterns for a period/segment at a given lead time.
+    Returns average pickup and pickup rate compared to final.
+    """
+    # Get column name for this lead time
+    column = f"d{lead_days}" if lead_days <= 30 else f"d{lead_days}"  # Use same format for all
+
+    # For lead times with pace data, use pace table
+    pace_type = 'resident' if is_resident else 'non_resident'
+
+    if lead_days <= 365:  # We have pace columns up to d365
+        result = await db.execute(
+            text(f"""
+                SELECT
+                    AVG(COALESCE({column}, 0)) as avg_at_lead,
+                    AVG(COALESCE(d0, 0)) as avg_final
+                FROM resos_booking_pace
+                WHERE pace_type = :pace_type
+                AND booking_date >= CURRENT_DATE - CAST(:lookback_days AS INTEGER)
+                AND booking_date < CURRENT_DATE
+            """),
+            {"pace_type": pace_type, "lookback_days": lookback_weeks * 7}
+        )
+    else:
+        # Use aggregated stats table for period-specific analysis
+        result = await db.execute(
+            text("""
+                SELECT
+                    AVG(CASE WHEN :is_resident THEN hotel_guest_covers ELSE non_hotel_guest_covers END) as avg_covers
+                FROM resos_bookings_stats
+                WHERE date >= CURRENT_DATE - CAST(:lookback_days AS INTEGER)
+                AND date < CURRENT_DATE
+            """),
+            {"is_resident": is_resident, "lookback_days": lookback_weeks * 7}
+        )
+
+    row = result.fetchone()
+
+    return {
+        "avg_at_lead": row.avg_at_lead if row and row.avg_at_lead else 0,
+        "avg_final": row.avg_final if row and row.avg_final else 0
+    }
+
+
+async def forecast_covers_for_date(
+    db: AsyncSession,
+    target_date: date,
+    include_details: bool = False
+) -> Dict[str, Any]:
+    """
+    Generate covers forecast for a specific date.
+
+    Returns breakdown by period and segment:
+    - Breakfast: Based on previous night's occupancy
+    - Lunch: OTB + non-resident pickup
+    - Dinner: OTB (resident + non-resident) + pickup for each
+    """
+    today = date.today()
+    lead_days = (target_date - today).days
+    prior_year_date = get_prior_year_date(target_date)
+
+    # Get current OTB covers
+    current_covers = await get_resos_covers_for_date(db, target_date)
+
+    # Get prior year covers
+    prior_covers = await get_resos_covers_for_date(db, prior_year_date)
+
+    # Get hotel occupancy for the night before (for breakfast)
+    night_before = target_date - timedelta(days=1)
+    prior_year_night_before = get_prior_year_date(night_before)
+
+    # Get current hotel OTB for night before
+    hotel_otb = await get_hotel_occupancy_for_date(db, night_before)
+    # Get prior year hotel occupancy for night before (tells us expected final)
+    hotel_prior = await get_hotel_occupancy_for_date(db, prior_year_night_before)
+
+    # Get breakfast rate (covers per room)
+    breakfast_rate = await get_historical_breakfast_rate(db)
+
+    # Calculate forecasts by period
+    result = {
+        "date": target_date.isoformat(),
+        "day_of_week": target_date.strftime("%a"),
+        "lead_days": lead_days,
+        "prior_year_date": prior_year_date.isoformat(),
+    }
+
+    # ============ BREAKFAST ============
+    # Breakfast = hotel guests from night before (guests eat breakfast, not rooms)
+    # Past: use actual hotel guest count
+    # Future: OTB guests + pickup from pickupv2 hotel forecast
+
+    hotel_guests_otb = hotel_otb["guests"]
+    hotel_rooms_otb = hotel_otb["occupied_rooms"]
+    hotel_guests_prior = hotel_prior["guests"]
+    hotel_rooms_prior = hotel_prior["occupied_rooms"]
+
+    # Calculate guests per room ratio for converting room forecast to guests
+    # Use prior year ratio (more stable/representative of final state) with fallbacks
+    if hotel_rooms_prior > 0:
+        guests_per_room = hotel_guests_prior / hotel_rooms_prior
+    elif hotel_rooms_otb > 0:
+        guests_per_room = hotel_guests_otb / hotel_rooms_otb
+    else:
+        guests_per_room = 1.8  # Default fallback
+
+    breakfast_calc = None
+    if lead_days <= 0:
+        # PAST: Use actual hotel guest count
+        breakfast_otb = hotel_guests_otb
+        breakfast_pickup = 0
+        breakfast_forecast = breakfast_otb
+    else:
+        # FUTURE: Use pickupv2 model for room forecast
+        breakfast_otb = hotel_guests_otb
+
+        # Get pickupv2 room forecast for the night before
+        # (night_before lead_days = lead_days for target_date since breakfast is next morning)
+        night_before_lead_days = lead_days - 1  # Night before has 1 less lead day
+        pickup_rooms = 0
+        try:
+            pickupv2_forecast = await forecast_rooms_for_date(
+                db,
+                night_before,
+                night_before_lead_days,
+                prior_year_night_before,
+                'hotel_room_nights'
+            )
+            if pickupv2_forecast:
+                # Get forecasted rooms and pickup from pickupv2
+                forecasted_rooms = pickupv2_forecast.get('predicted_value', hotel_rooms_otb)
+                pickup_rooms = pickupv2_forecast.get('pickup_rooms_total', 0)
+
+                # Convert pickup rooms to guests using the ratio (round up)
+                breakfast_pickup = math.ceil(pickup_rooms * guests_per_room)
+                # Forecast = OTB + pickup (floor is always OTB guests)
+                breakfast_forecast = breakfast_otb + breakfast_pickup
+
+                # Store calculation details
+                breakfast_calc = {
+                    "night_before": night_before.isoformat(),
+                    "hotel_rooms_otb": hotel_rooms_otb,
+                    "hotel_guests_otb": hotel_guests_otb,
+                    "pickup_rooms": round(pickup_rooms, 1),
+                    "guests_per_room": round(guests_per_room, 2),
+                    "source": "pickupv2",
+                }
+            else:
+                # Fallback to prior year pattern
+                breakfast_pickup = max(0, hotel_guests_prior - hotel_guests_otb)
+                breakfast_forecast = breakfast_otb + breakfast_pickup
+                breakfast_calc = {
+                    "night_before": night_before.isoformat(),
+                    "hotel_guests_prior": hotel_guests_prior,
+                    "source": "prior_year_fallback",
+                }
+        except Exception as e:
+            logger.warning(f"Pickupv2 forecast failed for {night_before}: {e}")
+            # Fallback to prior year pattern
+            breakfast_pickup = max(0, hotel_guests_prior - hotel_guests_otb)
+            breakfast_forecast = breakfast_otb + breakfast_pickup
+            breakfast_calc = {
+                "night_before": night_before.isoformat(),
+                "hotel_guests_prior": hotel_guests_prior,
+                "source": "prior_year_fallback",
+            }
+
+    # Prior year breakfast (for comparison)
+    prior_breakfast = hotel_guests_prior
+
+    result["breakfast"] = {
+        "otb": breakfast_otb,
+        "pickup": breakfast_pickup,
+        "forecast": breakfast_forecast,
+        "prior_year": prior_breakfast,
+        "hotel_guests_otb": hotel_guests_otb,
+        "hotel_guests_prior": hotel_guests_prior,
+        "calc": breakfast_calc,
+    }
+
+    # ============ LUNCH ============
+    # Lunch: OTB + pickup based on median historical pickup at lead time
+    # Uses straight pickup count (not ratio) for stability
+    lunch_data = current_covers.get("lunch", {})
+    lunch_otb = lunch_data.get("total_covers", 0)
+    prior_lunch = prior_covers.get("lunch", {}).get("total_covers", 0)
+
+    # Get median pickup count for this lead time and DOW
+    lunch_pickup = await get_lunch_pickup_by_lead_time(db, target_date, lead_days)
+
+    # Determine pace column for tooltip
+    if lead_days <= 30:
+        lunch_pace_col = f"d{lead_days}"
+    elif lead_days <= 177:
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        lunch_pace_col = f"d{min(weekly_cols, key=lambda x: abs(x - lead_days))}"
+    else:
+        lunch_pace_col = "d177"
+
+    lunch_calc = None
+    # For future dates, add pickup to OTB
+    if lead_days > 0:
+        lunch_forecast = lunch_otb + lunch_pickup
+        lunch_calc = {
+            "day_of_week": target_date.strftime("%A"),
+            "lead_days": lead_days,
+            "pace_column": lunch_pace_col,
+            "lookback_weeks": 8,
+            "median_pickup": lunch_pickup,
+            "source": "resos_booking_pace (total)",
+        }
+    else:
+        # Past date - no pickup
+        lunch_pickup = 0
+        lunch_forecast = lunch_otb
+
+    result["lunch"] = {
+        "otb": lunch_otb,
+        "pickup": lunch_pickup,
+        "forecast": lunch_forecast,
+        "prior_year": prior_lunch,
+        "calc": lunch_calc,
+    }
+
+    # ============ DINNER ============
+    # Dinner: More sophisticated calculation
+    # - Non-resident: Lead-time based median pickup (like lunch)
+    # - Resident: Based on hotel guests without dinner reservations + conversion rate
+    dinner_data = current_covers.get("dinner", {})
+    dinner_otb = dinner_data.get("total_covers", 0)
+    dinner_resident_otb = dinner_data.get("resident_covers", 0)
+    dinner_non_resident_otb = dinner_data.get("non_resident_covers", 0)
+    dinner_dbb_otb = dinner_data.get("dbb_covers", 0)
+
+    prior_dinner = prior_covers.get("dinner", {}).get("total_covers", 0)
+    prior_dinner_resident = prior_covers.get("dinner", {}).get("resident_covers", 0)
+    prior_dinner_non_resident = prior_covers.get("dinner", {}).get("non_resident_covers", 0)
+
+    # Determine pace column for non-resident tooltip
+    if lead_days <= 30:
+        dinner_pace_col = f"d{lead_days}"
+    elif lead_days <= 177:
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        dinner_pace_col = f"d{min(weekly_cols, key=lambda x: abs(x - lead_days))}"
+    else:
+        dinner_pace_col = "d177"
+
+    non_resident_calc = None
+    if lead_days > 0:
+        # ---- NON-RESIDENT PICKUP ----
+        # Use lead-time based median pickup (same logic as lunch)
+        non_resident_pickup = await get_dinner_non_resident_pickup_by_lead_time(db, target_date, lead_days)
+        non_resident_calc = {
+            "day_of_week": target_date.strftime("%A"),
+            "lead_days": lead_days,
+            "pace_column": dinner_pace_col,
+            "lookback_weeks": 8,
+            "median_pickup": non_resident_pickup,
+            "source": "resos_booking_pace (non_resident)",
+        }
+
+        # ---- RESIDENT PICKUP ----
+        # Simple approach: % of hotel guests who dine, applied to forecasted guests
+        # Get hotel occupancy for target_date (dinner is same night as stay)
+        hotel_tonight = await get_hotel_occupancy_for_date(db, target_date)
+        hotel_guests_otb = hotel_tonight["guests"]
+        hotel_rooms_otb = hotel_tonight["occupied_rooms"]
+
+        # Calculate guests per room (use prior year ratio if current is 0)
+        prior_year_hotel = await get_hotel_occupancy_for_date(db, prior_year_date)
+        if hotel_rooms_otb > 0:
+            guests_per_room = hotel_guests_otb / hotel_rooms_otb
+        elif prior_year_hotel["occupied_rooms"] > 0:
+            guests_per_room = prior_year_hotel["guests"] / prior_year_hotel["occupied_rooms"]
+        else:
+            guests_per_room = 1.8  # Default
+
+        # Get pickupv2 room forecast for tonight
+        pickup_rooms = 0
+        try:
+            pickupv2_dinner = await forecast_rooms_for_date(
+                db, target_date, lead_days, prior_year_date, 'hotel_room_nights'
+            )
+            if pickupv2_dinner:
+                pickup_rooms = pickupv2_dinner.get('pickup_rooms_total', 0)
+        except Exception as e:
+            logger.warning(f"Pickupv2 forecast failed for dinner {target_date}: {e}")
+
+        # Calculate forecasted hotel guests (OTB + pickup)
+        pickup_guests = pickup_rooms * guests_per_room
+        forecasted_guests = hotel_guests_otb + pickup_guests
+
+        # Get historical resident dining rate (% of hotel guests who dine)
+        dining_rate = await get_resident_dining_rate(db, target_date)
+
+        # Calculate expected resident covers
+        # forecasted_resident_covers = forecasted_guests × dining_rate
+        forecasted_resident_covers = forecasted_guests * dining_rate
+
+        # Resident pickup = expected total - current OTB resident covers
+        resident_pickup = max(0, math.ceil(forecasted_resident_covers) - dinner_resident_otb)
+
+        dinner_forecast = dinner_otb + resident_pickup + non_resident_pickup
+
+        # Store calculation details for tooltip
+        resident_calc = {
+            "hotel_guests_otb": hotel_guests_otb,
+            "pickup_rooms": round(pickup_rooms, 1),
+            "guests_per_room": round(guests_per_room, 2),
+            "pickup_guests": round(pickup_guests, 1),
+            "forecasted_guests": round(forecasted_guests, 1),
+            "dining_rate": round(dining_rate * 100, 1),  # As percentage
+            "forecasted_resident_covers": round(forecasted_resident_covers, 1),
+            "resident_otb": dinner_resident_otb,
+            "source": "last 4 weeks same DOW",
+        }
+    else:
+        # Past date - no pickup
+        dinner_forecast = dinner_otb
+        resident_pickup = 0
+        non_resident_pickup = 0
+        resident_calc = None
+        non_resident_calc = None
+
+    result["dinner"] = {
+        "otb": dinner_otb,
+        "resident_otb": dinner_resident_otb,
+        "non_resident_otb": dinner_non_resident_otb,
+        "dbb_otb": dinner_dbb_otb,
+        "resident_pickup": resident_pickup,
+        "non_resident_pickup": non_resident_pickup,
+        "forecast": dinner_forecast,
+        "prior_year": prior_dinner,
+        "prior_resident": prior_dinner_resident,
+        "prior_non_resident": prior_dinner_non_resident,
+        "resident_calc": resident_calc,
+        "non_resident_calc": non_resident_calc,
+    }
+
+    # Totals
+    total_otb = breakfast_otb + lunch_otb + dinner_otb
+    total_forecast = breakfast_forecast + lunch_forecast + dinner_forecast
+    prior_total = prior_breakfast + prior_lunch + prior_dinner
+
+    result["totals"] = {
+        "otb": total_otb,
+        "forecast": total_forecast,
+        "prior_year": prior_total,
+        "pace_vs_prior_pct": round((total_otb / prior_total * 100), 1) if prior_total > 0 else None
+    }
+
+    # Add hotel occupancy context
+    result["hotel_context"] = {
+        "night_before_occupancy": hotel_otb["occupancy_pct"],
+        "night_before_rooms": hotel_otb["occupied_rooms"],
+        "night_before_guests": hotel_otb["guests"]
+    }
+
+    return result
+
+
+async def forecast_covers_range(
+    db: AsyncSession,
+    start_date: date,
+    end_date: date,
+    include_details: bool = False
+) -> Dict[str, Any]:
+    """
+    Generate covers forecast for a date range.
+    """
+    forecasts = []
+    current = start_date
+
+    while current <= end_date:
+        try:
+            day_forecast = await forecast_covers_for_date(db, current, include_details)
+            forecasts.append(day_forecast)
+        except Exception as e:
+            logger.warning(f"Failed to forecast covers for {current}: {e}")
+
+        current += timedelta(days=1)
+
+    # Calculate summary
+    summary = {
+        "breakfast_otb": sum(f["breakfast"]["otb"] for f in forecasts),
+        "breakfast_forecast": sum(f["breakfast"]["forecast"] for f in forecasts),
+        "breakfast_prior": sum(f["breakfast"]["prior_year"] for f in forecasts),
+        "lunch_otb": sum(f["lunch"]["otb"] for f in forecasts),
+        "lunch_forecast": sum(f["lunch"]["forecast"] for f in forecasts),
+        "lunch_prior": sum(f["lunch"]["prior_year"] for f in forecasts),
+        "dinner_otb": sum(f["dinner"]["otb"] for f in forecasts),
+        "dinner_forecast": sum(f["dinner"]["forecast"] for f in forecasts),
+        "dinner_prior": sum(f["dinner"]["prior_year"] for f in forecasts),
+        "total_otb": sum(f["totals"]["otb"] for f in forecasts),
+        "total_forecast": sum(f["totals"]["forecast"] for f in forecasts),
+        "total_prior": sum(f["totals"]["prior_year"] for f in forecasts),
+        "days_count": len(forecasts)
+    }
+
+    return {
+        "data": forecasts,
+        "summary": summary
+    }

backend/services/forecasting/historical_forecast.py

@@ -0,0 +1,659 @@
+"""
+Historical Forecast Runner
+Runs all models as if it were a specific historical date.
+
+This allows backtesting of Prophet, XGBoost, and Pickup models
+by only using data that would have been available at that time.
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+import pandas as pd
+import numpy as np
+import json
+from sqlalchemy import text
+
+from utils.time_alignment import get_prior_year_daily
+
+logger = logging.getLogger(__name__)
+
+
+async def run_historical_forecast(
+    db,
+    simulated_today: date,
+    metric_codes: List[str] = None,
+    models: List[str] = None,
+    forecast_days: int = 60
+) -> dict:
+    """
+    Run forecasts as if today were a specific historical date.
+
+    Only uses data that would have been available on simulated_today.
+
+    Args:
+        db: Database session
+        simulated_today: The date to pretend "today" is
+        metric_codes: List of metrics to forecast (default: all main metrics)
+        models: List of models to run (default: all)
+        forecast_days: Number of days to forecast (default: 60)
+
+    Returns:
+        Dict with results summary
+    """
+    if metric_codes is None:
+        metric_codes = ['hotel_room_nights', 'hotel_occupancy_pct', 'resos_dinner_covers', 'resos_lunch_covers']
+
+    if models is None:
+        models = ['prophet', 'xgboost', 'pickup', 'catboost']
+
+    forecast_from = simulated_today + timedelta(days=1)
+    forecast_to = simulated_today + timedelta(days=forecast_days)
+
+    results = {
+        "simulated_today": str(simulated_today),
+        "forecast_from": str(forecast_from),
+        "forecast_to": str(forecast_to),
+        "metrics": {},
+        "total_forecasts": 0
+    }
+
+    for metric_code in metric_codes:
+        results["metrics"][metric_code] = {}
+
+        for model in models:
+            try:
+                if model == 'prophet':
+                    forecasts = await _run_prophet_historical(
+                        db, metric_code, simulated_today, forecast_from, forecast_to
+                    )
+                elif model == 'xgboost':
+                    forecasts = await _run_xgboost_historical(
+                        db, metric_code, simulated_today, forecast_from, forecast_to
+                    )
+                elif model == 'pickup':
+                    forecasts = await _run_pickup_historical(
+                        db, metric_code, simulated_today, forecast_from, forecast_to
+                    )
+                elif model == 'catboost':
+                    forecasts = await _run_catboost_historical(
+                        db, metric_code, simulated_today, forecast_from, forecast_to
+                    )
+                else:
+                    continue
+
+                results["metrics"][metric_code][model] = len(forecasts)
+                results["total_forecasts"] += len(forecasts)
+                await db.commit()  # Commit after each successful model run
+
+            except Exception as e:
+                logger.error(f"Historical {model} forecast failed for {metric_code}: {e}")
+                await db.rollback()  # Rollback on error to clear failed transaction
+                results["metrics"][metric_code][model] = f"error: {str(e)[:100]}"
+    logger.info(f"Historical forecasts complete for {simulated_today}: {results['total_forecasts']} total forecasts")
+
+    return results
+
+
+async def _run_prophet_historical(
+    db,
+    metric_code: str,
+    simulated_today: date,
+    forecast_from: date,
+    forecast_to: date,
+    training_days: int = 2555
+) -> List[dict]:
+    """Run Prophet using only data available before simulated_today."""
+    try:
+        from prophet import Prophet
+
+        # Get room capacity for capping (sum across all room categories for a single date)
+        total_rooms = 25
+        if metric_code in ('hotel_room_nights', 'hotel_occupancy_pct'):
+            rooms_result = await db.execute(
+                text("""
+                SELECT COALESCE(SUM(available), 25) as total_rooms
+                FROM newbook_occupancy_report
+                WHERE date = (
+                    SELECT MAX(date) FROM newbook_occupancy_report
+                    WHERE date <= :simulated_today
+                )
+                """),
+                {"simulated_today": simulated_today}
+            )
+            rooms_row = rooms_result.fetchone()
+            if rooms_row and rooms_row.total_rooms:
+                total_rooms = int(rooms_row.total_rooms)
+
+        # Training data ends at simulated_today - 1 (yesterday from simulated perspective)
+        training_to = simulated_today - timedelta(days=1)
+        training_from = training_to - timedelta(days=training_days)
+
+        result = await db.execute(
+            text("""
+            SELECT date, actual_value
+            FROM daily_metrics
+            WHERE metric_code = :metric_code
+                AND date BETWEEN :from_date AND :to_date
+                AND actual_value IS NOT NULL
+            ORDER BY date
+            """),
+            {"metric_code": metric_code, "from_date": training_from, "to_date": training_to}
+        )
+        rows = result.fetchall()
+
+        if len(rows) < 30:
+            logger.warning(f"Insufficient data for historical Prophet: {metric_code} has {len(rows)} records as of {simulated_today}")
+            return []
+
+        df = pd.DataFrame([{"ds": row.date, "y": float(row.actual_value)} for row in rows])
+
+        model = Prophet(
+            yearly_seasonality=True,
+            weekly_seasonality=True,
+            daily_seasonality=False,
+            interval_width=0.80
+        )
+        model.add_country_holidays(country_name='GB')
+        model.fit(df)
+
+        future_dates = pd.date_range(start=forecast_from, end=forecast_to, freq='D')
+        future_df = pd.DataFrame({"ds": future_dates})
+        forecast = model.predict(future_df)
+
+        forecasts = []
+        for _, row in forecast.iterrows():
+            predicted_value = float(row["yhat"])
+            lower_bound = float(row["yhat_lower"])
+            upper_bound = float(row["yhat_upper"])
+
+            # Apply physical caps
+            if metric_code == 'hotel_occupancy_pct':
+                predicted_value = min(predicted_value, 100)
+                lower_bound = min(lower_bound, 100)
+                upper_bound = min(upper_bound, 100)
+            if metric_code == 'hotel_room_nights':
+                predicted_value = min(predicted_value, total_rooms)
+                lower_bound = min(lower_bound, total_rooms)
+                upper_bound = min(upper_bound, total_rooms)
+
+            forecast_record = {
+                "forecast_date": row["ds"].date(),
+                "forecast_type": metric_code,
+                "model_type": "prophet",
+                "predicted_value": round(predicted_value, 2),
+                "lower_bound": round(lower_bound, 2),
+                "upper_bound": round(upper_bound, 2)
+            }
+            forecasts.append(forecast_record)
+
+            # Store with generated_at = simulated_today to track when this "would have been" generated
+            await db.execute(
+                text("""
+                INSERT INTO forecasts (
+                    forecast_date, forecast_type, model_type,
+                    predicted_value, lower_bound, upper_bound, generated_at
+                ) VALUES (
+                    :forecast_date, :forecast_type, :model_type,
+                    :predicted_value, :lower_bound, :upper_bound, :generated_at
+                )
+                """),
+                {**forecast_record, "generated_at": simulated_today}
+            )
+
+        logger.info(f"Historical Prophet forecast for {metric_code} as of {simulated_today}: {len(forecasts)} records")
+        return forecasts
+
+    except ImportError:
+        logger.error("Prophet not installed")
+        return []
+    except Exception as e:
+        logger.error(f"Historical Prophet failed: {e}")
+        raise
+
+
+async def _run_xgboost_historical(
+    db,
+    metric_code: str,
+    simulated_today: date,
+    forecast_from: date,
+    forecast_to: date,
+    training_days: int = 2555
+) -> List[dict]:
+    """Run XGBoost using only data available before simulated_today."""
+    try:
+        import xgboost as xgb
+
+        training_to = simulated_today - timedelta(days=1)
+        training_from = training_to - timedelta(days=training_days + 60)
+
+        result = await db.execute(
+            text("""
+            SELECT date, actual_value
+            FROM daily_metrics
+            WHERE metric_code = :metric_code
+                AND date BETWEEN :from_date AND :to_date
+                AND actual_value IS NOT NULL
+            ORDER BY date
+            """),
+            {"metric_code": metric_code, "from_date": training_from, "to_date": training_to}
+        )
+        rows = result.fetchall()
+
+        if len(rows) < 60:
+            logger.warning(f"Insufficient data for historical XGBoost: {metric_code} has {len(rows)} records")
+            return []
+
+        df = pd.DataFrame([{"ds": pd.Timestamp(row.date), "y": float(row.actual_value)} for row in rows])
+        df = df.sort_values('ds').reset_index(drop=True)
+        df = _create_features(df)
+        df = df.dropna()
+
+        feature_cols = [
+            'day_of_week', 'month', 'day_of_month', 'week_of_year', 'is_weekend',
+            'dow_sin', 'dow_cos', 'month_sin', 'month_cos',
+            'lag_7', 'lag_14', 'lag_21', 'lag_28',
+            'rolling_mean_7', 'rolling_mean_14', 'rolling_mean_28',
+            'rolling_std_7', 'rolling_std_14', 'rolling_std_28'
+        ]
+
+        if 'lag_365' in df.columns and df['lag_365'].notna().sum() > 30:
+            feature_cols.append('lag_365')
+
+        X = df[feature_cols]
+        y = df['y']
+
+        model = xgb.XGBRegressor(
+            n_estimators=100,
+            max_depth=5,
+            learning_rate=0.1,
+            objective='reg:squarederror',
+            random_state=42
+        )
+        model.fit(X, y)
+
+        forecasts = []
+        current_df = df.copy()
+
+        for forecast_date in pd.date_range(start=forecast_from, end=forecast_to, freq='D'):
+            new_row = pd.DataFrame([{"ds": forecast_date, "y": np.nan}])
+            current_df = pd.concat([current_df, new_row], ignore_index=True)
+            current_df = _create_features(current_df)
+
+            X_pred = current_df[feature_cols].iloc[-1:].ffill()
+            prediction = float(model.predict(X_pred)[0])
+            current_df.iloc[-1, current_df.columns.get_loc('y')] = prediction
+
+            forecast_record = {
+                "forecast_date": forecast_date.date(),
+                "forecast_type": metric_code,
+                "model_type": "xgboost",
+                "predicted_value": round(float(prediction), 2)
+            }
+            forecasts.append(forecast_record)
+
+            await db.execute(
+                text("""
+                INSERT INTO forecasts (
+                    forecast_date, forecast_type, model_type, predicted_value, generated_at
+                ) VALUES (
+                    :forecast_date, :forecast_type, :model_type, :predicted_value, :generated_at
+                )
+                """),
+                {**forecast_record, "generated_at": simulated_today}
+            )
+
+        logger.info(f"Historical XGBoost forecast for {metric_code} as of {simulated_today}: {len(forecasts)} records")
+        return forecasts
+
+    except ImportError as e:
+        logger.error(f"Required package not installed: {e}")
+        return []
+    except Exception as e:
+        logger.error(f"Historical XGBoost failed: {e}")
+        raise
+
+
+async def _run_pickup_historical(
+    db,
+    metric_code: str,
+    simulated_today: date,
+    forecast_from: date,
+    forecast_to: date
+) -> List[dict]:
+    """Run Pickup model using only data available before simulated_today."""
+
+    forecasts = []
+
+    # Get room capacity (sum of available rooms across all categories for a single date)
+    total_rooms = 25
+    if metric_code in ('hotel_room_nights', 'hotel_occupancy_pct'):
+        rooms_result = await db.execute(
+            text("""
+            SELECT COALESCE(SUM(available), 25) as total_rooms
+            FROM newbook_occupancy_report
+            WHERE date = (
+                SELECT MAX(date) FROM newbook_occupancy_report
+                WHERE date <= :simulated_today
+            )
+            """),
+            {"simulated_today": simulated_today}
+        )
+        rooms_row = rooms_result.fetchone()
+        if rooms_row and rooms_row.total_rooms:
+            total_rooms = int(rooms_row.total_rooms)
+
+    for days_out in range((forecast_to - forecast_from).days + 1):
+        forecast_date = forecast_from + timedelta(days=days_out)
+        lead_time = (forecast_date - simulated_today).days
+
+        if lead_time < 1:
+            continue
+
+        # Calculate prior year comparison date
+        prior_year_date = get_prior_year_daily(forecast_date)
+
+        # Get/reconstruct current OTB as of simulated_today
+        current_otb = await _get_reconstructed_otb(
+            db, metric_code, forecast_date, simulated_today, total_rooms
+        )
+
+        if current_otb is None:
+            continue
+
+        # Get prior year OTB at same lead time
+        prior_simulated_today = get_prior_year_daily(simulated_today)
+        prior_otb = await _get_reconstructed_otb(
+            db, metric_code, prior_year_date, prior_simulated_today, total_rooms
+        )
+
+        # Get prior year final actual
+        prior_final_result = await db.execute(
+            text("""
+            SELECT actual_value
+            FROM daily_metrics
+            WHERE date = :prior_date AND metric_code = :metric
+            """),
+            {"prior_date": prior_year_date, "metric": metric_code}
+        )
+        prior_final_row = prior_final_result.fetchone()
+        prior_final = float(prior_final_row.actual_value) if prior_final_row and prior_final_row.actual_value else None
+
+        # Calculate projection using additive method
+        projected_value = current_otb
+
+        if prior_otb is not None and prior_final is not None:
+            prior_pickup = prior_final - prior_otb
+            projected_value = current_otb + prior_pickup
+
+            if projected_value < current_otb:
+                projected_value = current_otb
+
+            # Apply caps
+            if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                projected_value = 100
+            if metric_code == 'hotel_room_nights' and projected_value > total_rooms:
+                projected_value = total_rooms
+
+        elif prior_final is not None and prior_final > 0:
+            # Implied additive
+            if lead_time >= 28:
+                estimated_pct = 0.35
+            elif lead_time >= 14:
+                estimated_pct = 0.55
+            elif lead_time >= 7:
+                estimated_pct = 0.75
+            else:
+                estimated_pct = 0.90
+
+            implied_pickup = prior_final * (1 - estimated_pct)
+            projected_value = current_otb + implied_pickup
+            projected_value = max(projected_value, current_otb)
+
+            if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                projected_value = 100
+            if metric_code == 'hotel_room_nights' and projected_value > total_rooms:
+                projected_value = total_rooms
+
+        forecast_record = {
+            "forecast_date": forecast_date,
+            "forecast_type": metric_code,
+            "model_type": "pickup",
+            "predicted_value": round(projected_value, 2)
+        }
+        forecasts.append(forecast_record)
+
+        await db.execute(
+            text("""
+            INSERT INTO forecasts (
+                forecast_date, forecast_type, model_type, predicted_value, generated_at
+            ) VALUES (
+                :forecast_date, :forecast_type, :model_type, :predicted_value, :generated_at
+            )
+            """),
+            {**forecast_record, "generated_at": simulated_today}
+        )
+
+    logger.info(f"Historical Pickup forecast for {metric_code} as of {simulated_today}: {len(forecasts)} records")
+    return forecasts
+
+
+async def _run_catboost_historical(
+    db,
+    metric_code: str,
+    simulated_today: date,
+    forecast_from: date,
+    forecast_to: date,
+    training_days: int = 2555
+) -> List[dict]:
+    """Run CatBoost using only data available before simulated_today."""
+    try:
+        from catboost import CatBoostRegressor
+
+        training_to = simulated_today - timedelta(days=1)
+        training_from = training_to - timedelta(days=training_days + 60)
+
+        result = await db.execute(
+            text("""
+            SELECT date, actual_value
+            FROM daily_metrics
+            WHERE metric_code = :metric_code
+                AND date BETWEEN :from_date AND :to_date
+                AND actual_value IS NOT NULL
+            ORDER BY date
+            """),
+            {"metric_code": metric_code, "from_date": training_from, "to_date": training_to}
+        )
+        rows = result.fetchall()
+
+        if len(rows) < 60:
+            logger.warning(f"Insufficient data for historical CatBoost: {metric_code} has {len(rows)} records")
+            return []
+
+        df = pd.DataFrame([{"ds": pd.Timestamp(row.date), "y": float(row.actual_value)} for row in rows])
+        df = df.sort_values('ds').reset_index(drop=True)
+        df = _create_catboost_features(df)
+        df = df.dropna()
+
+        categorical_features = ['day_of_week', 'month']
+        numerical_features = [
+            'day_of_month', 'week_of_year', 'is_weekend',
+            'lag_7', 'lag_14', 'lag_21', 'lag_28',
+            'rolling_mean_7', 'rolling_mean_14', 'rolling_mean_28',
+            'rolling_std_7', 'rolling_std_14', 'rolling_std_28'
+        ]
+
+        if 'lag_365' in df.columns and df['lag_365'].notna().sum() > 30:
+            numerical_features.append('lag_365')
+
+        feature_cols = categorical_features + numerical_features
+
+        X = df[feature_cols]
+        y = df['y']
+
+        model = CatBoostRegressor(
+            iterations=200,
+            depth=6,
+            learning_rate=0.1,
+            loss_function='RMSE',
+            cat_features=categorical_features,
+            verbose=False,
+            random_seed=42
+        )
+        model.fit(X, y)
+
+        forecasts = []
+        current_df = df.copy()
+
+        for forecast_date in pd.date_range(start=forecast_from, end=forecast_to, freq='D'):
+            new_row = pd.DataFrame([{"ds": forecast_date, "y": np.nan}])
+            current_df = pd.concat([current_df, new_row], ignore_index=True)
+            current_df = _create_catboost_features(current_df)
+
+            X_pred = current_df[feature_cols].iloc[-1:].copy()
+            for col in numerical_features:
+                if col in X_pred.columns:
+                    X_pred[col] = X_pred[col].ffill()
+                    if X_pred[col].isna().any():
+                        X_pred[col] = X_pred[col].fillna(0)
+
+            prediction = float(model.predict(X_pred)[0])
+            prediction = max(0, prediction)
+            current_df.iloc[-1, current_df.columns.get_loc('y')] = prediction
+
+            forecast_record = {
+                "forecast_date": forecast_date.date(),
+                "forecast_type": metric_code,
+                "model_type": "catboost",
+                "predicted_value": round(float(prediction), 2)
+            }
+            forecasts.append(forecast_record)
+
+            await db.execute(
+                text("""
+                INSERT INTO forecasts (
+                    forecast_date, forecast_type, model_type, predicted_value, generated_at
+                ) VALUES (
+                    :forecast_date, :forecast_type, :model_type, :predicted_value, :generated_at
+                )
+                """),
+                {**forecast_record, "generated_at": simulated_today}
+            )
+
+        logger.info(f"Historical CatBoost forecast for {metric_code} as of {simulated_today}: {len(forecasts)} records")
+        return forecasts
+
+    except ImportError as e:
+        logger.error(f"CatBoost not installed: {e}")
+        return []
+    except Exception as e:
+        logger.error(f"Historical CatBoost failed: {e}")
+        raise
+
+
+def _create_catboost_features(df: pd.DataFrame) -> pd.DataFrame:
+    """Create features for CatBoost model with native categorical support."""
+    df = df.copy()
+
+    df['day_of_week'] = df['ds'].dt.dayofweek.astype(str)  # Categorical for CatBoost
+    df['month'] = df['ds'].dt.month.astype(str)  # Categorical for CatBoost
+    df['day_of_month'] = df['ds'].dt.day
+    df['week_of_year'] = df['ds'].dt.isocalendar().week.astype(int)
+    df['is_weekend'] = (df['ds'].dt.dayofweek >= 5).astype(int)
+
+    for lag in [7, 14, 21, 28]:
+        df[f'lag_{lag}'] = df['y'].shift(lag)
+
+    for window in [7, 14, 28]:
+        df[f'rolling_mean_{window}'] = df['y'].rolling(window=window, min_periods=1).mean()
+        df[f'rolling_std_{window}'] = df['y'].rolling(window=window, min_periods=1).std().fillna(0)
+
+    if len(df) > 365:
+        df['lag_365'] = df['y'].shift(365)
+
+    return df
+
+
+async def _get_reconstructed_otb(
+    db,
+    metric_code: str,
+    target_date: date,
+    as_of_date: date,
+    total_rooms: int = 25
+) -> Optional[float]:
+    """
+    Get or reconstruct OTB value for a target date as of a specific date.
+
+    First tries pickup_snapshots, then reconstructs from booking data.
+    """
+    # Try snapshots first
+    snap_result = await db.execute(
+        text("""
+        SELECT otb_value
+        FROM pickup_snapshots
+        WHERE stay_date = :target_date
+            AND metric_type = :metric
+            AND snapshot_date <= :as_of_date
+        ORDER BY snapshot_date DESC
+        LIMIT 1
+        """),
+        {"target_date": target_date, "metric": metric_code, "as_of_date": as_of_date}
+    )
+    snap_row = snap_result.fetchone()
+
+    if snap_row and snap_row.otb_value is not None:
+        return float(snap_row.otb_value)
+
+    # Reconstruct from booking data
+    # EXCLUDES overflow category (category_id=5) used for chargeable no-shows
+    if metric_code in ('hotel_room_nights', 'hotel_occupancy_pct'):
+        # Use CAST instead of :: to avoid asyncpg parameter parsing issues
+        recon_result = await db.execute(
+            text("""
+            SELECT COUNT(DISTINCT newbook_id) as otb_count
+            FROM newbook_bookings
+            WHERE arrival_date <= :target_date
+                AND departure_date > :target_date
+                AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')
+                AND CAST(raw_json->>'booking_placed' AS timestamp) <= CAST(:as_of_date AS date) + INTERVAL '1 day'
+                AND (category_id IS NULL OR category_id != '5')
+            """),
+            {"target_date": target_date, "as_of_date": as_of_date}
+        )
+        recon_row = recon_result.fetchone()
+
+        if recon_row:
+            otb_count = recon_row.otb_count or 0
+            if metric_code == 'hotel_occupancy_pct':
+                return (otb_count / total_rooms) * 100 if total_rooms > 0 else 0
+            else:
+                return otb_count
+
+    return None
+
+
+def _create_features(df: pd.DataFrame) -> pd.DataFrame:
+    """Create features for XGBoost model."""
+    df = df.copy()
+
+    df['day_of_week'] = df['ds'].dt.dayofweek
+    df['month'] = df['ds'].dt.month
+    df['day_of_month'] = df['ds'].dt.day
+    df['week_of_year'] = df['ds'].dt.isocalendar().week.astype(int)
+    df['is_weekend'] = (df['day_of_week'] >= 5).astype(int)
+
+    df['dow_sin'] = np.sin(2 * np.pi * df['day_of_week'] / 7)
+    df['dow_cos'] = np.cos(2 * np.pi * df['day_of_week'] / 7)
+    df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12)
+    df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)
+
+    for lag in [7, 14, 21, 28]:
+        df[f'lag_{lag}'] = df['y'].shift(lag)
+
+    for window in [7, 14, 28]:
+        df[f'rolling_mean_{window}'] = df['y'].rolling(window=window, min_periods=1).mean()
+        df[f'rolling_std_{window}'] = df['y'].rolling(window=window, min_periods=1).std()
+
+    if len(df) > 365:
+        df['lag_365'] = df['y'].shift(365)
+
+    return df

backend/services/forecasting/pickup_model.py

@@ -0,0 +1,332 @@
+"""
+Pickup forecasting model
+Hotel industry standard pace/pickup tracking
+Compares current on-the-books vs historical patterns
+
+Uses ADDITIVE method for small properties:
+- Projected = current_otb + expected_pickup_count
+- Where expected_pickup_count = prior_year_final - prior_year_otb
+
+This avoids ratio distortion with small numbers (e.g., 2→6 = 3x ratio
+applied to 5 = 15 rooms, which is unrealistic)
+
+Prior year comparison uses 364 days (52 weeks) for day-of-week alignment:
+- Monday compares to Monday
+- Saturday compares to Saturday
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+from sqlalchemy import text
+
+from utils.time_alignment import get_prior_year_daily, get_comparison_info
+from utils.capacity import get_bookable_cap_sync
+
+logger = logging.getLogger(__name__)
+
+
+async def run_pickup_forecast(
+    db,
+    metric_code: str,
+    forecast_from: date,
+    forecast_to: date
+) -> List[dict]:
+    """
+    Run Pickup model forecast for a metric
+
+    The pickup model works by:
+    1. Getting current on-the-books (OTB) for each future date
+    2. Comparing to prior year SAME DAY OF WEEK at same lead time
+    3. Projecting final using ADDITIVE method (not ratio) for reliability
+
+    Additive method: projected = current_otb + (prior_final - prior_otb)
+    This represents: "what I have now + what typically picks up from here"
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast
+        forecast_from: Start date for forecasts
+        forecast_to: End date for forecasts
+
+    Returns:
+        List of forecast records
+    """
+    forecasts = []
+    today = date.today()
+
+    for days_out in range((forecast_to - forecast_from).days + 1):
+        forecast_date = forecast_from + timedelta(days=days_out)
+        lead_time = (forecast_date - today).days
+
+        if lead_time < 1:
+            continue  # Can't do pickup for past dates
+
+        # Calculate prior year comparison date (same day of week alignment)
+        prior_year_date = get_prior_year_daily(forecast_date)
+
+        # Get bookable rooms for this date (rooms - maintenance, for capping)
+        bookable_cap = get_bookable_cap_sync(db, forecast_date, fallback_value=25)
+
+        # Get current OTB from snapshots
+        otb_result = db.execute(
+            text("""
+            SELECT otb_value, prior_year_otb, prior_year_final
+            FROM pickup_snapshots
+            WHERE stay_date = :forecast_date
+                AND metric_type = :metric_code
+                AND snapshot_date = :today
+            """),
+            {"forecast_date": forecast_date, "metric_code": metric_code, "today": today}
+        )
+        otb_row = otb_result.fetchone()
+
+        if not otb_row:
+            # No OTB data available, skip
+            continue
+
+        current_otb = float(otb_row.otb_value) if otb_row.otb_value is not None else 0
+        # Use 'is not None' - 0 is valid data meaning no bookings at that lead time
+        prior_otb = float(otb_row.prior_year_otb) if otb_row.prior_year_otb is not None else None
+        prior_final = float(otb_row.prior_year_final) if otb_row.prior_year_final is not None else None
+
+        # Get pickup curve for this day of week and season (fallback)
+        day_of_week = forecast_date.weekday()
+        day_name = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][day_of_week]
+        month = forecast_date.month
+
+        # Determine season
+        if month in [6, 7, 8]:
+            season = 'peak'
+        elif month in [12, 1, 2]:
+            season = 'low'
+        else:
+            season = 'shoulder'
+
+        curve_result = db.execute(
+            text("""
+            SELECT avg_pct_of_final, std_dev
+            FROM pickup_curves
+            WHERE day_of_week = :dow
+                AND season = :season
+                AND metric_type = :metric_code
+                AND days_out = :lead_time
+            """),
+            {"dow": day_of_week, "season": season, "metric_code": metric_code, "lead_time": lead_time}
+        )
+        curve_row = curve_result.fetchone()
+
+        # Calculate projection
+        projected_value = current_otb
+        projection_method = 'current_otb'
+        pace_vs_prior = None
+        confidence_note = "Using current on-the-books"
+
+        if prior_otb is not None and prior_final is not None:
+            # Calculate expected pickup count from prior year
+            prior_pickup_count = prior_final - prior_otb  # How many picked up from this lead time
+
+            # Calculate pace vs prior year
+            if prior_otb > 0:
+                pace_vs_prior = ((current_otb - prior_otb) / prior_otb) * 100
+
+            # ADDITIVE METHOD: current + expected pickup
+            # This is more reliable for small properties than ratio method
+            # Example: prior had 2 OTB → 6 final = 4 pickup
+            #          current has 5 OTB → project 5 + 4 = 9
+            projected_value = current_otb + prior_pickup_count
+
+            # Ensure projection is at least current OTB (pickup can't be negative in projection)
+            if projected_value < current_otb:
+                projected_value = current_otb
+                projection_method = 'additive_floor'
+                confidence_note = f"vs {day_name} {prior_year_date.strftime('%d %b %Y')}: {prior_otb:.0f}→{prior_final:.0f} (negative pickup, using OTB)"
+            else:
+                projection_method = 'additive'
+                confidence_note = f"vs {day_name} {prior_year_date.strftime('%d %b %Y')}: {prior_otb:.0f}→{prior_final:.0f} (+{prior_pickup_count:.0f} pickup)"
+
+            # Apply physical caps
+            if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                projected_value = 100
+                confidence_note += " (capped at 100%)"
+            if metric_code == 'hotel_room_nights' and projected_value > bookable_cap:
+                projected_value = bookable_cap
+                confidence_note += f" (capped at {bookable_cap} bookable rooms)"
+
+        elif prior_final is not None and prior_final > 0:
+            # No prior OTB, but have prior final - use as guidance
+            # Estimate typical OTB percentage at this lead time
+            if lead_time >= 28:
+                estimated_pct = 0.35
+            elif lead_time >= 14:
+                estimated_pct = 0.55
+            elif lead_time >= 7:
+                estimated_pct = 0.75
+            else:
+                estimated_pct = 0.90
+
+            # Calculate implied pickup from typical percentages
+            implied_prior_otb = prior_final * estimated_pct
+            implied_pickup = prior_final - implied_prior_otb
+
+            # Apply additive method with implied pickup
+            projected_value = current_otb + implied_pickup
+            projected_value = max(projected_value, current_otb)
+
+            projection_method = 'implied_additive'
+            confidence_note = f"vs {day_name} {prior_year_date.strftime('%d %b %Y')}: final was {prior_final:.0f}, est +{implied_pickup:.0f} pickup at {lead_time}d"
+
+            # Apply physical caps
+            if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                projected_value = 100
+                confidence_note += " (capped at 100%)"
+            if metric_code == 'hotel_room_nights' and projected_value > bookable_cap:
+                projected_value = bookable_cap
+                confidence_note += f" (capped at {bookable_cap} bookable rooms)"
+
+        elif curve_row and curve_row.avg_pct_of_final > 0:
+            # Curve method: project based on historical pickup curve
+            projected_value = current_otb / (curve_row.avg_pct_of_final / 100)
+            projection_method = 'curve'
+            confidence_note = f"Based on pickup curve ({curve_row.avg_pct_of_final:.1f}% typical at {lead_time} days out)"
+
+            # Apply physical caps
+            if metric_code == 'hotel_occupancy_pct' and projected_value > 100:
+                projected_value = 100
+                confidence_note += " (capped at 100%)"
+            if metric_code == 'hotel_room_nights' and projected_value > bookable_cap:
+                projected_value = bookable_cap
+                confidence_note += f" (capped at {bookable_cap} bookable rooms)"
+
+        forecast_record = {
+            "forecast_date": forecast_date,
+            "forecast_type": metric_code,
+            "model_type": "pickup",
+            "predicted_value": round(projected_value, 2)
+        }
+        forecasts.append(forecast_record)
+
+        # Store in database
+        db.execute(
+            text("""
+            INSERT INTO forecasts (
+                forecast_date, forecast_type, model_type, predicted_value, generated_at
+            ) VALUES (
+                :forecast_date, :forecast_type, :model_type, :predicted_value, NOW()
+            )
+            """),
+            forecast_record
+        )
+
+        # Store explanation
+        try:
+            db.execute(
+                text("""
+                INSERT INTO pickup_explanations (
+                    forecast_date, forecast_type, current_otb, days_out,
+                    comparison_otb, comparison_final,
+                    pickup_curve_pct, pace_vs_prior_pct, projection_method,
+                    projected_value, confidence_note, generated_at
+                ) VALUES (
+                    :date, :metric, :otb, :days_out,
+                    :prior_otb, :prior_final,
+                    :curve_pct, :pace, :method,
+                    :projected, :confidence, NOW()
+                )
+                """),
+                {
+                    "date": forecast_date,
+                    "metric": metric_code,
+                    "otb": current_otb,
+                    "days_out": lead_time,
+                    "prior_otb": prior_otb,
+                    "prior_final": prior_final,
+                    "curve_pct": curve_row.avg_pct_of_final if curve_row else None,
+                    "pace": pace_vs_prior,
+                    "method": projection_method,
+                    "projected": projected_value,
+                    "confidence": confidence_note
+                }
+            )
+        except Exception:
+            pass  # Skip if conflict, explanations are supplementary
+
+    db.commit()
+    logger.info(f"Pickup forecast generated for {metric_code}: {len(forecasts)} records")
+    return forecasts
+
+
+async def update_pickup_curves(db, metric_code: str, lookback_days: int = 2555):
+    """
+    Update historical pickup curves from actuals
+
+    Calculates average percentage of final value at each lead time
+    """
+    logger.info(f"Updating pickup curves for {metric_code}")
+
+    # For each day of week and season
+    for dow in range(7):
+        for season in ['peak', 'shoulder', 'low']:
+            # Get historical final values and snapshots
+            result = db.execute(
+                text("""
+                WITH final_values AS (
+                    SELECT date, actual_value
+                    FROM daily_metrics
+                    WHERE metric_code = :metric_code
+                        AND date > CURRENT_DATE - :lookback
+                        AND actual_value IS NOT NULL
+                        AND EXTRACT(DOW FROM date) = :dow
+                ),
+                snapshot_data AS (
+                    SELECT
+                        ps.stay_date,
+                        ps.days_out,
+                        ps.otb_value,
+                        fv.actual_value as final_value
+                    FROM pickup_snapshots ps
+                    JOIN final_values fv ON ps.stay_date = fv.date
+                    WHERE ps.metric_type = :metric_code
+                )
+                SELECT
+                    days_out,
+                    AVG(otb_value / NULLIF(final_value, 0) * 100) as avg_pct,
+                    STDDEV(otb_value / NULLIF(final_value, 0) * 100) as std_pct,
+                    COUNT(*) as sample_count
+                FROM snapshot_data
+                WHERE final_value > 0
+                GROUP BY days_out
+                HAVING COUNT(*) >= 5
+                """),
+                {"metric_code": metric_code, "lookback": lookback_days, "dow": dow}
+            )
+
+            for row in result.fetchall():
+                db.execute(
+                    text("""
+                    INSERT INTO pickup_curves (
+                        day_of_week, season, metric_type, days_out,
+                        avg_pct_of_final, std_dev, sample_count, updated_at
+                    ) VALUES (
+                        :dow, :season, :metric, :days_out,
+                        :avg_pct, :std, :count, NOW()
+                    )
+                    ON CONFLICT (day_of_week, season, metric_type, days_out)
+                    DO UPDATE SET
+                        avg_pct_of_final = :avg_pct,
+                        std_dev = :std,
+                        sample_count = :count,
+                        updated_at = NOW()
+                    """),
+                    {
+                        "dow": dow,
+                        "season": season,
+                        "metric": metric_code,
+                        "days_out": row.days_out,
+                        "avg_pct": row.avg_pct,
+                        "std": row.std_pct,
+                        "count": row.sample_count
+                    }
+                )
+
+    db.commit()
+    logger.info(f"Pickup curves updated for {metric_code}")

backend/services/forecasting/pickup_tuned.py

@@ -0,0 +1,177 @@
+"""
+Pickup Tuned Model Service
+
+This is the production-tuned Pickup model extracted from the preview endpoint.
+Uses the exact same logic as the frontend preview to ensure value consistency.
+
+Pickup Formula: Forecast = Current OTB + (Prior Year Final - Prior Year OTB)
+
+This transparent model calculates expected pickup based on prior year booking patterns.
+Only works for room-based metrics (occupancy, rooms). Not applicable to revenue metrics.
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Dict, Optional
+from sqlalchemy import text
+
+from utils.capacity import get_bookable_cap
+
+logger = logging.getLogger(__name__)
+
+
+def get_lead_time_column(lead_days: int) -> str:
+    """Map lead days to the appropriate column in newbook_booking_pace."""
+    if lead_days <= 0:
+        return "d0"
+    elif lead_days <= 30:
+        return f"d{lead_days}"
+    elif lead_days <= 177:
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        for col in weekly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d177"
+    else:
+        monthly_cols = [210, 240, 270, 300, 330, 365]
+        for col in monthly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d365"
+
+
+async def run_pickup_tuned_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    perception_date: Optional[date] = None
+) -> List[Dict]:
+    """
+    Generate Pickup forecast using production-tuned model.
+
+    Uses transparent booking pace formula:
+    Forecast = Current OTB + (Prior Year Final - Prior Year OTB)
+
+    This uses the exact same logic as the preview endpoint to ensure
+    backend snapshots match frontend preview values.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast (only room-based metrics supported)
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        perception_date: Optional date to generate forecast as-of (for backtesting)
+
+    Returns:
+        List of forecast dicts with forecast_date and predicted_value
+    """
+    logger.info(f"Running Pickup tuned forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Map metric codes to preview endpoint metric names
+    metric_map = {
+        'hotel_occupancy_pct': 'occupancy',
+        'hotel_room_nights': 'rooms',
+        'hotel_guests': 'guests',
+    }
+
+    metric = metric_map.get(metric_code, 'rooms')
+
+    # Check if metric is room-based (pickup model only works for these)
+    is_room_based = metric in ('occupancy', 'rooms')
+    if not is_room_based:
+        logger.warning(f"Pickup model doesn't apply to non-room metric: {metric_code}")
+        return []
+
+    # Use perception_date if provided, otherwise use actual today
+    today = perception_date if perception_date else date.today()
+
+    # Get default bookable cap
+    default_bookable_cap = await get_bookable_cap(db)
+
+    # Generate forecasts for each date
+    forecasts = []
+
+    current_date = start_date
+    while current_date <= end_date:
+        lead_days = (current_date - today).days
+        if lead_days < 0:
+            current_date += timedelta(days=1)
+            continue
+
+        lead_col = get_lead_time_column(lead_days)
+        prior_year_date = current_date - timedelta(days=364)  # 52 weeks for DOW alignment
+
+        # Get current OTB
+        current_query = text("""
+            SELECT booking_count as current_otb
+            FROM newbook_bookings_stats
+            WHERE date = :arrival_date
+        """)
+        current_result = await db.execute(current_query, {"arrival_date": current_date})
+        current_row = current_result.fetchone()
+
+        # Get prior year OTB from booking_pace (for lead time comparison)
+        prior_year_for_otb = current_date - timedelta(days=364)
+        prior_otb_query = text(f"""
+            SELECT {lead_col} as prior_otb
+            FROM newbook_booking_pace
+            WHERE arrival_date = :prior_date
+        """)
+        prior_otb_result = await db.execute(prior_otb_query, {"prior_date": prior_year_for_otb})
+        prior_otb_row = prior_otb_result.fetchone()
+
+        # Get prior year FINAL from bookings_stats
+        prior_final_query = text("""
+            SELECT booking_count as prior_final
+            FROM newbook_bookings_stats
+            WHERE date = :prior_date
+        """)
+        prior_final_result = await db.execute(prior_final_query, {"prior_date": prior_year_date})
+        prior_final_row = prior_final_result.fetchone()
+
+        # Extract values
+        current_otb = current_row.current_otb if current_row and current_row.current_otb is not None else 0
+        prior_otb = prior_otb_row.prior_otb if prior_otb_row and prior_otb_row.prior_otb is not None else None
+        prior_final = prior_final_row.prior_final if prior_final_row and prior_final_row.prior_final is not None else 0
+
+        # Get per-date bookable cap
+        date_bookable_cap = await get_bookable_cap(db, current_date, default_bookable_cap)
+
+        # Convert to occupancy % if metric is occupancy
+        if metric == "occupancy" and date_bookable_cap > 0:
+            if current_otb is not None:
+                current_otb = (current_otb / date_bookable_cap) * 100
+            if prior_otb is not None:
+                prior_otb = (prior_otb / date_bookable_cap) * 100
+            if prior_final is not None:
+                prior_final = (prior_final / date_bookable_cap) * 100
+
+        # Calculate forecast using pickup formula
+        forecast = None
+
+        if current_otb is not None:
+            if prior_final is not None and prior_otb is not None:
+                expected_pickup = prior_final - prior_otb
+                forecast = current_otb + expected_pickup
+                # Floor to current OTB if pickup is negative
+                if forecast < current_otb:
+                    forecast = current_otb
+                # Cap at max capacity (uses per-date bookable cap)
+                if metric == "occupancy" and forecast > 100:
+                    forecast = 100.0
+                elif metric == "rooms" and forecast > date_bookable_cap:
+                    forecast = float(date_bookable_cap)
+            else:
+                # No prior year data - use current OTB as forecast
+                forecast = current_otb
+
+        if forecast is not None:
+            forecasts.append({
+                'forecast_date': current_date,
+                'predicted_value': round(forecast, 1)
+            })
+
+        current_date += timedelta(days=1)
+
+    logger.info(f"Pickup tuned generated {len(forecasts)} forecasts for {metric_code}")
+    return forecasts

backend/services/forecasting/prophet_model.py

@@ -0,0 +1,195 @@
+"""
+Prophet forecasting model
+Time series forecasting with trend, seasonality, and holiday effects
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+import pandas as pd
+import numpy as np
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+async def run_prophet_forecast(
+    db,
+    metric_code: str,
+    forecast_from: date,
+    forecast_to: date,
+    training_days: int = 2555  # ~7 years - use all available history
+) -> List[dict]:
+    """
+    Run Prophet forecast for a metric
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast (e.g., 'hotel_occupancy_pct')
+        forecast_from: Start date for forecasts
+        forecast_to: End date for forecasts
+        training_days: Days of historical data to use for training
+
+    Returns:
+        List of forecast records
+    """
+    try:
+        from prophet import Prophet
+
+        # Get historical data
+        training_from = forecast_from - timedelta(days=training_days)
+
+        # Revenue metrics use earned_revenue_data joined with gl_accounts
+        revenue_metrics = ['net_accom', 'net_dry', 'net_wet', 'total_rev']
+        if metric_code in revenue_metrics:
+            revenue_departments = {
+                'net_accom': 'accommodation',
+                'net_dry': 'dry',
+                'net_wet': 'wet',
+                'total_rev': None,  # All departments
+            }
+            department = revenue_departments.get(metric_code)
+            if department is None and metric_code != 'total_rev':
+                logger.warning(f"Unknown revenue metric for Prophet: {metric_code}")
+                return []
+
+            if metric_code == 'total_rev':
+                # Total revenue across all departments
+                result = db.execute(
+                    text("""
+                    SELECT date, SUM(amount_net) as actual_value
+                    FROM newbook_earned_revenue_data
+                    WHERE date BETWEEN :from_date AND :to_date
+                    GROUP BY date
+                    HAVING SUM(amount_net) IS NOT NULL
+                    ORDER BY date
+                    """),
+                    {"from_date": training_from, "to_date": forecast_from - timedelta(days=1)}
+                )
+            else:
+                # Revenue by department
+                result = db.execute(
+                    text("""
+                    SELECT r.date, SUM(r.amount_net) as actual_value
+                    FROM newbook_earned_revenue_data r
+                    JOIN newbook_gl_accounts g ON r.gl_account_id = g.gl_account_id
+                    WHERE r.date BETWEEN :from_date AND :to_date
+                        AND g.department = :department
+                    GROUP BY r.date
+                    HAVING SUM(r.amount_net) IS NOT NULL
+                    ORDER BY r.date
+                    """),
+                    {"from_date": training_from, "to_date": forecast_from - timedelta(days=1), "department": department}
+                )
+        else:
+            # Hotel metrics use newbook_bookings_stats table
+            metric_column_map = {
+                'hotel_occupancy_pct': 'total_occupancy_pct',
+                'hotel_room_nights': 'booking_count',
+                'hotel_guests': 'guests_count',
+            }
+
+            column_name = metric_column_map.get(metric_code)
+            if not column_name:
+                logger.warning(f"Unknown metric_code for Prophet: {metric_code}")
+                return []
+
+            result = db.execute(
+                text(f"""
+                SELECT date, {column_name} as actual_value
+                FROM newbook_bookings_stats
+                WHERE date BETWEEN :from_date AND :to_date
+                    AND {column_name} IS NOT NULL
+                ORDER BY date
+                """),
+                {"from_date": training_from, "to_date": forecast_from - timedelta(days=1)}
+            )
+
+        rows = result.fetchall()
+
+        if len(rows) < 30:
+            logger.warning(f"Insufficient data for Prophet forecast: {metric_code} has {len(rows)} records")
+            return []
+
+        # Prepare data for Prophet
+        df = pd.DataFrame([{"ds": row.date, "y": float(row.actual_value)} for row in rows])
+
+        # Initialize and fit Prophet model
+        model = Prophet(
+            yearly_seasonality=True,
+            weekly_seasonality=True,
+            daily_seasonality=False,
+            interval_width=0.80  # 80% confidence interval
+        )
+
+        # Add UK holidays
+        model.add_country_holidays(country_name='GB')
+
+        model.fit(df)
+
+        # Generate future dates
+        future_dates = pd.date_range(start=forecast_from, end=forecast_to, freq='D')
+        future_df = pd.DataFrame({"ds": future_dates})
+
+        # Make predictions
+        forecast = model.predict(future_df)
+
+        # Store forecasts
+        forecasts = []
+        for _, row in forecast.iterrows():
+            forecast_record = {
+                "forecast_date": row["ds"].date(),
+                "forecast_type": metric_code,
+                "model_type": "prophet",
+                "predicted_value": round(float(row["yhat"]), 2),
+                "lower_bound": round(float(row["yhat_lower"]), 2),
+                "upper_bound": round(float(row["yhat_upper"]), 2)
+            }
+            forecasts.append(forecast_record)
+
+            # Store in database - simple insert (latest value wins)
+            db.execute(
+                text("""
+                INSERT INTO forecasts (
+                    forecast_date, forecast_type, model_type,
+                    predicted_value, lower_bound, upper_bound, generated_at
+                ) VALUES (
+                    :forecast_date, :forecast_type, :model_type,
+                    :predicted_value, :lower_bound, :upper_bound, NOW()
+                )
+                """),
+                forecast_record
+            )
+
+            # Store decomposition for explainability
+            # Simple insert - decomposition stored per generation
+            try:
+                db.execute(
+                    text("""
+                    INSERT INTO prophet_decomposition (
+                        forecast_date, forecast_type, trend,
+                        yearly_seasonality, weekly_seasonality, generated_at
+                    ) VALUES (
+                        :date, :metric, :trend, :yearly, :weekly, NOW()
+                    )
+                    """),
+                    {
+                        "date": row["ds"].date(),
+                        "metric": metric_code,
+                        "trend": round(float(row.get("trend", 0)), 4),
+                        "yearly": round(float(row.get("yearly", 0)), 4),
+                        "weekly": round(float(row.get("weekly", 0)), 4)
+                    }
+                )
+            except Exception:
+                pass  # Skip if conflict, decomposition is supplementary
+
+        db.commit()
+        logger.info(f"Prophet forecast generated for {metric_code}: {len(forecasts)} records")
+        return forecasts
+
+    except ImportError:
+        logger.error("Prophet not installed. Install with: pip install prophet")
+        return []
+    except Exception as e:
+        logger.error(f"Prophet forecast failed for {metric_code}: {e}")
+        return []

backend/services/forecasting/prophet_tuned.py

@@ -0,0 +1,281 @@
+"""
+Prophet Tuned Model Service
+
+This is the production-tuned Prophet model extracted from the prophet-preview endpoint.
+Uses the exact same logic as the frontend preview to ensure value consistency.
+
+Features:
+- 2 years of training data
+- Logistic growth with floor/cap
+- UK holidays + custom special dates
+- OTB floor capping
+- Per-date bookable cap adjustments
+- Metric-specific handling
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Dict, Optional
+import pandas as pd
+import warnings
+from prophet import Prophet
+from sqlalchemy import text
+
+from utils.capacity import get_bookable_cap
+
+logger = logging.getLogger(__name__)
+warnings.filterwarnings('ignore')
+
+
+# Metric configuration mapping
+METRIC_COLUMN_MAP = {
+    'occupancy': ('s.occupancy_pct', False, True),
+    'rooms': ('s.booking_count', False, False),
+    'guests': ('s.guest_count', False, False),
+    'ave_guest_rate': ('s.arr_net', False, False),
+    'arr': ('s.arr_net', False, False),
+    'net_accom': ('r.accommodation', True, False),
+    'net_dry': ('r.dry', True, False),
+    'net_wet': ('r.wet', True, False),
+    'total_rev': ('(COALESCE(r.accommodation, 0) + COALESCE(r.dry, 0) + COALESCE(r.wet, 0))', True, False),
+}
+
+
+def get_metric_query_parts(metric: str) -> tuple:
+    """
+    Get SQL query parts for a metric.
+    Returns: (column_expr, from_clause, is_percentage)
+    """
+    if metric not in METRIC_COLUMN_MAP:
+        # Default to rooms if unknown metric
+        metric = 'rooms'
+
+    col_expr, needs_revenue, is_pct = METRIC_COLUMN_MAP[metric]
+
+    if needs_revenue:
+        from_clause = """
+            FROM newbook_bookings_stats s
+            LEFT JOIN newbook_net_revenue_data r ON s.date = r.date
+        """
+    else:
+        from_clause = "FROM newbook_bookings_stats s"
+
+    return col_expr, from_clause, is_pct
+
+
+def get_lead_time_column(lead_days: int) -> str:
+    """
+    Map lead days to the appropriate column in newbook_booking_pace.
+    """
+    if lead_days <= 0:
+        return "d0"
+    elif lead_days <= 30:
+        return f"d{lead_days}"
+    elif lead_days <= 177:
+        # Weekly intervals - find nearest column
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        for col in weekly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d177"
+    else:
+        # Monthly intervals
+        monthly_cols = [210, 240, 270, 300, 330, 365]
+        for col in monthly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d365"
+
+
+async def run_prophet_tuned_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    perception_date: Optional[date] = None
+) -> List[Dict]:
+    """
+    Generate Prophet forecast using production-tuned model.
+
+    This uses the exact same logic as the prophet-preview endpoint to ensure
+    backend snapshots match frontend preview values.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast (e.g., 'hotel_occupancy_pct', 'hotel_room_nights')
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        perception_date: Optional date to generate forecast as-of (for backtesting)
+
+    Returns:
+        List of forecast dicts with forecast_date and predicted_value
+    """
+    logger.info(f"Running Prophet tuned forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Map metric codes to preview endpoint metric names
+    metric_map = {
+        'hotel_occupancy_pct': 'occupancy',
+        'hotel_room_nights': 'rooms',
+        'hotel_guests': 'guests',
+        'hotel_arr': 'arr',
+        'ave_guest_rate': 'ave_guest_rate',
+        'net_accom': 'net_accom',
+        'net_dry': 'net_dry',
+        'net_wet': 'net_wet',
+        'total_rev': 'total_rev',
+    }
+
+    metric = metric_map.get(metric_code, 'rooms')
+
+    # Use perception_date if provided, otherwise use actual today
+    today = perception_date if perception_date else date.today()
+
+    # Get default bookable cap
+    default_bookable_cap = await get_bookable_cap(db)
+
+    # Get metric column and query parts
+    col_expr, from_clause, is_pct_metric = get_metric_query_parts(metric)
+
+    # Get historical data for Prophet training (past 2 years)
+    history_start = today - timedelta(days=730)
+    history_query = f"""
+        SELECT s.date as ds, {col_expr} as y
+        {from_clause}
+        WHERE s.date >= :history_start
+        AND s.date < :today
+        AND {col_expr} IS NOT NULL
+        ORDER BY s.date
+    """
+    history_result = await db.execute(text(history_query), {"history_start": history_start, "today": today})
+    history_rows = history_result.fetchall()
+
+    if len(history_rows) < 30:
+        logger.warning(f"Insufficient historical data for Prophet model: {len(history_rows)} rows")
+        return []
+
+    # Build training dataframe
+    df = pd.DataFrame([{"ds": row.ds, "y": float(row.y) if row.y is not None else 0} for row in history_rows])
+
+    # Set floor/cap based on metric type
+    if is_pct_metric:
+        # Percentage metrics (occupancy)
+        training_cap = 100
+    elif metric == 'rooms':
+        # Room counts - cap at bookable rooms
+        training_cap = default_bookable_cap
+    elif metric == 'guests':
+        # Guests can exceed rooms (multiple per room) - use historical max * 1.5
+        training_cap = df["y"].max() * 1.5 if len(df) > 0 and df["y"].max() > 0 else default_bookable_cap * 3
+    else:
+        # Revenue/rate metrics - use percentile-based cap
+        training_cap = df["y"].quantile(0.99) * 1.5 if len(df) > 0 and df["y"].quantile(0.99) > 0 else 10000
+
+    df["floor"] = 0
+    df["cap"] = training_cap
+
+    # Train Prophet model with logistic growth (respects floor/cap)
+    model = Prophet(
+        growth='logistic',
+        yearly_seasonality=True,
+        weekly_seasonality=True,
+        daily_seasonality=False,
+        interval_width=0.8,
+        changepoint_prior_scale=0.05
+    )
+
+    # Add UK holidays
+    model.add_country_holidays(country_name='UK')
+
+    # Add custom special dates from settings
+    try:
+        from api.special_dates import get_special_dates_for_prophet
+        # Get special dates for training period + forecast period
+        min_year = history_start.year
+        max_year = end_date.year + 1
+        custom_holidays = await get_special_dates_for_prophet(db, min_year, max_year)
+
+        if custom_holidays:
+            # Create holidays dataframe for Prophet
+            holidays_df = pd.DataFrame(custom_holidays)
+            # Group by holiday name and add lower/upper windows
+            for holiday_name in holidays_df['holiday'].unique():
+                holiday_dates = holidays_df[holidays_df['holiday'] == holiday_name][['ds', 'holiday']]
+                holiday_dates = holiday_dates.copy()
+                holiday_dates['lower_window'] = 0
+                holiday_dates['upper_window'] = 0
+                model.holidays = pd.concat([model.holidays, holiday_dates]) if model.holidays is not None else holiday_dates
+    except Exception as e:
+        logger.warning(f"Could not load special dates for Prophet: {e}")
+
+    model.fit(df)
+
+    # Create future dataframe for forecast period
+    future_dates = []
+    current_date = start_date
+    while current_date <= end_date:
+        if (current_date - today).days >= 0:
+            future_dates.append({"ds": current_date})
+        current_date += timedelta(days=1)
+
+    if not future_dates:
+        logger.warning("No future dates to forecast")
+        return []
+
+    future_df = pd.DataFrame(future_dates)
+
+    # Add floor/cap for logistic growth predictions (must match training cap)
+    future_df["floor"] = 0
+    future_df["cap"] = training_cap
+
+    forecast = model.predict(future_df)
+
+    # Process forecast results
+    forecasts = []
+    is_room_based = metric in ('occupancy', 'rooms')
+
+    for _, row in forecast.iterrows():
+        forecast_date = row["ds"].date()
+        lead_days = (forecast_date - today).days
+        lead_col = get_lead_time_column(lead_days)
+
+        # Get current OTB (only for room-based metrics)
+        current_otb = None
+        if is_room_based:
+            current_query = text("""
+                SELECT booking_count as current_otb
+                FROM newbook_bookings_stats
+                WHERE date = :arrival_date
+            """)
+            current_result = await db.execute(current_query, {"arrival_date": forecast_date})
+            current_row = current_result.fetchone()
+            current_otb = current_row.current_otb if current_row and current_row.current_otb is not None else 0
+
+        # Get per-date bookable cap for room-based metrics
+        date_bookable_cap = await get_bookable_cap(db, forecast_date, default_bookable_cap)
+
+        # Convert to occupancy if needed
+        if metric == "occupancy" and date_bookable_cap > 0:
+            if current_otb is not None:
+                current_otb = (current_otb / date_bookable_cap) * 100
+
+        # Get Prophet forecast values
+        yhat = row["yhat"]
+
+        # Cap at max capacity based on metric type (uses per-date bookable cap)
+        if is_pct_metric:
+            yhat = min(yhat, 100.0)
+        elif metric == 'rooms':
+            yhat = min(yhat, float(date_bookable_cap))
+        # Guests and revenue/rate metrics don't have a hard cap
+
+        # Floor forecast to current OTB if we have it (room-based metrics only)
+        # But never exceed the bookable capacity (e.g., closed/maintenance periods)
+        if is_room_based and current_otb is not None and yhat < current_otb:
+            yhat = min(current_otb, float(date_bookable_cap))
+
+        forecasts.append({
+            'forecast_date': forecast_date,
+            'predicted_value': round(yhat, 2)
+        })
+
+    logger.info(f"Prophet tuned generated {len(forecasts)} forecasts for {metric_code}")
+    return forecasts

backend/services/forecasting/xgboost_model.py

@@ -0,0 +1,300 @@
+"""
+XGBoost forecasting model
+Gradient boosting with feature engineering and SHAP explainability
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+import pandas as pd
+import numpy as np
+import json
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+def create_features(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Create features for XGBoost model
+
+    Features include:
+    - Day of week (0-6)
+    - Month (1-12)
+    - Day of month
+    - Week of year
+    - Is weekend
+    - Is holiday (would need holiday calendar)
+    - Lag features (7, 14, 28 days)
+    - Rolling averages (7, 14, 28 days)
+    """
+    df = df.copy()
+
+    # Date features
+    df['day_of_week'] = df['ds'].dt.dayofweek
+    df['month'] = df['ds'].dt.month
+    df['day_of_month'] = df['ds'].dt.day
+    df['week_of_year'] = df['ds'].dt.isocalendar().week.astype(int)
+    df['is_weekend'] = (df['day_of_week'] >= 5).astype(int)
+
+    # Cyclical encoding for day of week
+    df['dow_sin'] = np.sin(2 * np.pi * df['day_of_week'] / 7)
+    df['dow_cos'] = np.cos(2 * np.pi * df['day_of_week'] / 7)
+
+    # Cyclical encoding for month
+    df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12)
+    df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)
+
+    # Lag features
+    for lag in [7, 14, 21, 28]:
+        df[f'lag_{lag}'] = df['y'].shift(lag)
+
+    # Rolling averages
+    for window in [7, 14, 28]:
+        df[f'rolling_mean_{window}'] = df['y'].rolling(window=window, min_periods=1).mean()
+        df[f'rolling_std_{window}'] = df['y'].rolling(window=window, min_periods=1).std()
+
+    # Year-over-year feature: uses 365 days (same calendar date, not DOW-aligned)
+    # This is intentional for ML: captures date-specific patterns like holidays
+    # Combined with day_of_week features, the model learns both patterns
+    # Note: For direct comparisons (pickup model), use 364 days for DOW alignment
+    if len(df) > 365:
+        df['lag_365'] = df['y'].shift(365)
+
+    return df
+
+
+async def run_xgboost_forecast(
+    db,
+    metric_code: str,
+    forecast_from: date,
+    forecast_to: date,
+    training_days: int = 2555  # ~7 years - use all available history
+) -> List[dict]:
+    """
+    Run XGBoost forecast for a metric
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast
+        forecast_from: Start date for forecasts
+        forecast_to: End date for forecasts
+        training_days: Days of historical data to use
+
+    Returns:
+        List of forecast records
+    """
+    try:
+        import xgboost as xgb
+        from sklearn.model_selection import train_test_split
+
+        # Get historical data
+        training_from = forecast_from - timedelta(days=training_days + 60)  # Extra for lag features
+
+        # Revenue metrics use earned_revenue_data joined with gl_accounts
+        revenue_metrics = ['net_accom', 'net_dry', 'net_wet', 'total_rev']
+        if metric_code in revenue_metrics:
+            revenue_departments = {
+                'net_accom': 'accommodation',
+                'net_dry': 'dry',
+                'net_wet': 'wet',
+                'total_rev': None,  # All departments
+            }
+            department = revenue_departments.get(metric_code)
+            if department is None and metric_code != 'total_rev':
+                logger.warning(f"Unknown revenue metric for XGBoost: {metric_code}")
+                return []
+
+            if metric_code == 'total_rev':
+                # Total revenue across all departments
+                result = db.execute(
+                    text("""
+                    SELECT date, SUM(amount_net) as actual_value
+                    FROM newbook_earned_revenue_data
+                    WHERE date BETWEEN :from_date AND :to_date
+                    GROUP BY date
+                    HAVING SUM(amount_net) IS NOT NULL
+                    ORDER BY date
+                    """),
+                    {"from_date": training_from, "to_date": forecast_from - timedelta(days=1)}
+                )
+            else:
+                # Revenue by department
+                result = db.execute(
+                    text("""
+                    SELECT r.date, SUM(r.amount_net) as actual_value
+                    FROM newbook_earned_revenue_data r
+                    JOIN newbook_gl_accounts g ON r.gl_account_id = g.gl_account_id
+                    WHERE r.date BETWEEN :from_date AND :to_date
+                        AND g.department = :department
+                    GROUP BY r.date
+                    HAVING SUM(r.amount_net) IS NOT NULL
+                    ORDER BY r.date
+                    """),
+                    {"from_date": training_from, "to_date": forecast_from - timedelta(days=1), "department": department}
+                )
+        else:
+            # Hotel metrics use newbook_bookings_stats table
+            metric_column_map = {
+                'hotel_occupancy_pct': 'total_occupancy_pct',
+                'hotel_room_nights': 'booking_count',
+                'hotel_guests': 'guests_count',
+            }
+
+            column_name = metric_column_map.get(metric_code)
+            if not column_name:
+                logger.warning(f"Unknown metric_code for XGBoost: {metric_code}")
+                return []
+
+            result = db.execute(
+                text(f"""
+                SELECT date, {column_name} as actual_value
+                FROM newbook_bookings_stats
+                WHERE date BETWEEN :from_date AND :to_date
+                    AND {column_name} IS NOT NULL
+                ORDER BY date
+                """),
+                {"from_date": training_from, "to_date": forecast_from - timedelta(days=1)}
+            )
+
+        rows = result.fetchall()
+
+        if len(rows) < 60:
+            logger.warning(f"Insufficient data for XGBoost forecast: {metric_code} has {len(rows)} records")
+            return []
+
+        # Prepare data
+        df = pd.DataFrame([{"ds": pd.Timestamp(row.date), "y": float(row.actual_value)} for row in rows])
+        df = df.sort_values('ds').reset_index(drop=True)
+
+        # Create features
+        df = create_features(df)
+
+        # Remove rows with NaN from lag features
+        df = df.dropna()
+
+        # Define feature columns
+        feature_cols = [
+            'day_of_week', 'month', 'day_of_month', 'week_of_year', 'is_weekend',
+            'dow_sin', 'dow_cos', 'month_sin', 'month_cos',
+            'lag_7', 'lag_14', 'lag_21', 'lag_28',
+            'rolling_mean_7', 'rolling_mean_14', 'rolling_mean_28',
+            'rolling_std_7', 'rolling_std_14', 'rolling_std_28'
+        ]
+
+        # Add lag_365 if available
+        if 'lag_365' in df.columns and df['lag_365'].notna().sum() > 30:
+            feature_cols.append('lag_365')
+
+        X = df[feature_cols]
+        y = df['y']
+
+        # Train model
+        model = xgb.XGBRegressor(
+            n_estimators=100,
+            max_depth=5,
+            learning_rate=0.1,
+            objective='reg:squarederror',
+            random_state=42
+        )
+        model.fit(X, y)
+
+        # Generate forecasts
+        forecasts = []
+        current_df = df.copy()
+
+        for forecast_date in pd.date_range(start=forecast_from, end=forecast_to, freq='D'):
+            # Create row for forecast date
+            new_row = pd.DataFrame([{"ds": forecast_date, "y": np.nan}])
+            current_df = pd.concat([current_df, new_row], ignore_index=True)
+            current_df = create_features(current_df)
+
+            # Get features for prediction
+            X_pred = current_df[feature_cols].iloc[-1:].ffill()
+
+            # Make prediction
+            prediction = float(model.predict(X_pred)[0])
+
+            # Update y value for lag features
+            current_df.iloc[-1, current_df.columns.get_loc('y')] = prediction
+
+            forecast_record = {
+                "forecast_date": forecast_date.date(),
+                "forecast_type": metric_code,
+                "model_type": "xgboost",
+                "predicted_value": round(float(prediction), 2)
+            }
+            forecasts.append(forecast_record)
+
+            # Store in database
+            db.execute(
+                text("""
+                INSERT INTO forecasts (
+                    forecast_date, forecast_type, model_type, predicted_value, generated_at
+                ) VALUES (
+                    :forecast_date, :forecast_type, :model_type, :predicted_value, NOW()
+                )
+                """),
+                forecast_record
+            )
+
+        # Commit forecasts before SHAP calculations
+        db.commit()
+
+        # Calculate SHAP values for explainability
+        try:
+            import shap
+            explainer = shap.TreeExplainer(model)
+
+            # Get SHAP values for last few predictions
+            for i, forecast_date in enumerate(pd.date_range(start=forecast_from, end=min(forecast_from + timedelta(days=7), forecast_to), freq='D')):
+                idx = len(df) + i
+                X_explain = current_df[feature_cols].iloc[idx:idx+1].ffill()
+                shap_values = explainer.shap_values(X_explain)
+
+                # Store SHAP explanation
+                feature_contributions = dict(zip(feature_cols, shap_values[0].tolist()))
+                top_positive = sorted(
+                    [{"feature": k, "contribution": v} for k, v in feature_contributions.items() if v > 0],
+                    key=lambda x: x["contribution"], reverse=True
+                )[:5]
+                top_negative = sorted(
+                    [{"feature": k, "contribution": v} for k, v in feature_contributions.items() if v < 0],
+                    key=lambda x: x["contribution"]
+                )[:3]
+
+                try:
+                    db.execute(
+                        text("""
+                        INSERT INTO xgboost_explanations (
+                            forecast_date, forecast_type, base_value,
+                            feature_values, shap_values, top_positive, top_negative, generated_at
+                        ) VALUES (
+                            :date, :metric, :base_value, :feature_values,
+                            :shap_values, :top_positive, :top_negative, NOW()
+                        )
+                        """),
+                        {
+                            "date": forecast_date.date(),
+                            "metric": metric_code,
+                            "base_value": float(explainer.expected_value),
+                            "feature_values": json.dumps(X_explain.iloc[0].to_dict()),
+                            "shap_values": json.dumps(feature_contributions),
+                            "top_positive": json.dumps(top_positive),
+                            "top_negative": json.dumps(top_negative)
+                        }
+                    )
+                except Exception:
+                    pass  # Skip if conflict, explanations are supplementary
+        except Exception as e:
+            logger.warning(f"SHAP calculation failed: {e}")
+
+        db.commit()
+        logger.info(f"XGBoost forecast generated for {metric_code}: {len(forecasts)} records")
+        return forecasts
+
+    except ImportError as e:
+        logger.error(f"Required package not installed: {e}")
+        return []
+    except Exception as e:
+        logger.error(f"XGBoost forecast failed for {metric_code}: {e}")
+        return []

backend/services/forecasting/xgboost_tuned.py

@@ -0,0 +1,446 @@
+"""
+XGBoost Tuned Model Service
+
+This is the production-tuned XGBoost model extracted from the xgboost-preview endpoint.
+Uses the exact same logic as the frontend preview to ensure value consistency.
+
+Features:
+- 2 years of training data
+- Pace features (OTB at different lead times) for room-based metrics
+- Time-based features (day of week, month, week, weekend, special dates)
+- Lag features from prior year same DOW
+- OTB floor capping
+- Per-date bookable cap adjustments
+"""
+import logging
+from datetime import date, timedelta
+from typing import List, Dict, Optional
+import pandas as pd
+import numpy as np
+from xgboost import XGBRegressor
+import warnings
+from sqlalchemy import text
+
+from utils.capacity import get_bookable_cap
+from api.special_dates import resolve_special_date
+
+logger = logging.getLogger(__name__)
+warnings.filterwarnings('ignore')
+
+
+# Metric configuration mapping
+METRIC_COLUMN_MAP = {
+    'occupancy': ('s.occupancy_pct', False, True),
+    'rooms': ('s.booking_count', False, False),
+    'guests': ('s.guest_count', False, False),
+    'ave_guest_rate': ('s.arr_net', False, False),
+    'arr': ('s.arr_net', False, False),
+    'net_accom': ('r.accommodation', True, False),
+    'net_dry': ('r.dry', True, False),
+    'net_wet': ('r.wet', True, False),
+    'total_rev': ('(COALESCE(r.accommodation, 0) + COALESCE(r.dry, 0) + COALESCE(r.wet, 0))', True, False),
+}
+
+
+def get_metric_query_parts(metric: str) -> tuple:
+    """Get SQL query parts for a metric. Returns: (column_expr, from_clause, is_percentage)"""
+    if metric not in METRIC_COLUMN_MAP:
+        metric = 'rooms'
+    col_expr, needs_revenue, is_pct = METRIC_COLUMN_MAP[metric]
+    if needs_revenue:
+        from_clause = """
+            FROM newbook_bookings_stats s
+            LEFT JOIN newbook_net_revenue_data r ON s.date = r.date
+        """
+    else:
+        from_clause = "FROM newbook_bookings_stats s"
+    return col_expr, from_clause, is_pct
+
+
+def get_lead_time_column(lead_days: int) -> str:
+    """Map lead days to the appropriate column in newbook_booking_pace."""
+    if lead_days <= 0:
+        return "d0"
+    elif lead_days <= 30:
+        return f"d{lead_days}"
+    elif lead_days <= 177:
+        weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
+        for col in weekly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d177"
+    else:
+        monthly_cols = [210, 240, 270, 300, 330, 365]
+        for col in monthly_cols:
+            if lead_days <= col:
+                return f"d{col}"
+        return "d365"
+
+
+def round_towards_reference(value: float, reference: Optional[float]) -> int:
+    """Round a forecast value towards a reference value (prior year actual)."""
+    if reference is None:
+        return round(value)
+    if value < reference:
+        return int(np.ceil(value))
+    else:
+        return int(np.floor(value))
+
+
+async def run_xgboost_tuned_forecast(
+    db,
+    metric_code: str,
+    start_date: date,
+    end_date: date,
+    perception_date: Optional[date] = None
+) -> List[Dict]:
+    """
+    Generate XGBoost forecast using production-tuned model.
+
+    This uses the exact same logic as the xgboost-preview endpoint to ensure
+    backend snapshots match frontend preview values.
+
+    Args:
+        db: Database session
+        metric_code: Metric to forecast
+        start_date: Start date for forecast
+        end_date: End date for forecast
+        perception_date: Optional date to generate forecast as-of (for backtesting)
+
+    Returns:
+        List of forecast dicts with forecast_date and predicted_value
+    """
+    logger.info(f"Running XGBoost tuned forecast for {metric_code}: {start_date} to {end_date}")
+
+    # Map metric codes to preview endpoint metric names
+    metric_map = {
+        'hotel_occupancy_pct': 'occupancy',
+        'hotel_room_nights': 'rooms',
+        'hotel_guests': 'guests',
+        'hotel_arr': 'arr',
+        'ave_guest_rate': 'ave_guest_rate',
+        'net_accom': 'net_accom',
+        'net_dry': 'net_dry',
+        'net_wet': 'net_wet',
+        'total_rev': 'total_rev',
+    }
+
+    metric = metric_map.get(metric_code, 'rooms')
+
+    # Use perception_date if provided, otherwise use actual today
+    today = perception_date if perception_date else date.today()
+
+    # Get default bookable cap
+    default_bookable_cap = await get_bookable_cap(db)
+
+    # Get metric column and query parts
+    col_expr, from_clause, is_pct_metric = get_metric_query_parts(metric)
+    is_room_based = metric in ('occupancy', 'rooms')
+
+    # Get historical data for XGBoost training (past 2 years)
+    history_start = today - timedelta(days=730)
+
+    # Lead times to train on (key intervals) - only used for room-based metrics
+    train_lead_times = [0, 1, 3, 7, 14, 21, 28, 30]
+
+    # Get final values (and pace data for room-based metrics)
+    if is_room_based:
+        history_result = await db.execute(text("""
+            SELECT s.date as ds, s.booking_count as final,
+                   p.d0, p.d1, p.d3, p.d7, p.d14, p.d21, p.d28, p.d30
+            FROM newbook_bookings_stats s
+            LEFT JOIN newbook_booking_pace p ON s.date = p.arrival_date
+            WHERE s.date >= :history_start
+            AND s.date < :today
+            AND s.booking_count IS NOT NULL
+            ORDER BY s.date
+        """), {"history_start": history_start, "today": today})
+    else:
+        # Non-room metrics: get values without pace join
+        history_query = f"""
+            SELECT s.date as ds, {col_expr} as final
+            {from_clause}
+            WHERE s.date >= :history_start
+            AND s.date < :today
+            AND {col_expr} IS NOT NULL
+            ORDER BY s.date
+        """
+        history_result = await db.execute(text(history_query), {"history_start": history_start, "today": today})
+
+    history_rows = history_result.fetchall()
+
+    if len(history_rows) < 30:
+        logger.warning(f"Insufficient historical data for XGBoost model: {len(history_rows)} rows")
+        return []
+
+    # Load special dates for feature
+    special_date_set = set()
+    try:
+        special_dates_result = await db.execute(text(
+            "SELECT * FROM special_dates WHERE is_active = TRUE"
+        ))
+        special_dates_rows = special_dates_result.fetchall()
+        years_needed = set(r.ds.year for r in history_rows) | {today.year, today.year + 1}
+        for row in special_dates_rows:
+            sd = {
+                'pattern_type': row.pattern_type,
+                'fixed_month': row.fixed_month,
+                'fixed_day': row.fixed_day,
+                'nth_week': row.nth_week,
+                'weekday': row.weekday,
+                'month': row.month,
+                'relative_to_month': row.relative_to_month,
+                'relative_to_day': row.relative_to_day,
+                'relative_weekday': row.relative_weekday,
+                'relative_direction': row.relative_direction,
+                'duration_days': row.duration_days,
+                'is_recurring': row.is_recurring,
+                'one_off_year': row.one_off_year
+            }
+            for year in years_needed:
+                resolved_dates = resolve_special_date(sd, year)
+                for d in resolved_dates:
+                    special_date_set.add(d)
+    except Exception as e:
+        logger.warning(f"Could not load special dates: {e}")
+
+    # Build lookup dicts
+    final_by_date = {}
+    pace_by_date = {}
+    for row in history_rows:
+        final_by_date[row.ds] = row.final
+        if is_room_based and hasattr(row, 'd0'):
+            pace_by_date[row.ds] = {
+                0: row.d0, 1: row.d1, 3: row.d3, 7: row.d7,
+                14: row.d14, 21: row.d21, 28: row.d28, 30: row.d30
+            }
+
+    # Build training examples
+    training_rows = []
+
+    if is_room_based:
+        # Room-based metrics: use pace features (one per date,lead_time combo)
+        for row in history_rows:
+            ds = row.ds
+            final = float(row.final) if row.final else 0
+            prior_ds = ds - timedelta(days=364)
+
+            prior_final = final_by_date.get(prior_ds)
+            if prior_final is None:
+                continue
+
+            for lead_time in train_lead_times:
+                current_otb = pace_by_date.get(ds, {}).get(lead_time)
+                if current_otb is None:
+                    continue
+
+                prior_otb = pace_by_date.get(prior_ds, {}).get(lead_time)
+                if prior_otb is None:
+                    prior_otb = 0
+
+                otb_pct_of_prior_final = (float(current_otb) / float(prior_final) * 100) if prior_final > 0 else 0
+
+                training_rows.append({
+                    'ds': ds,
+                    'y': final,
+                    'days_out': lead_time,
+                    'current_otb': float(current_otb),
+                    'prior_otb_same_lead': float(prior_otb),
+                    'lag_364': float(prior_final),
+                    'otb_pct_of_prior_final': otb_pct_of_prior_final
+                })
+    else:
+        # Non-room metrics: use time features only (one per date)
+        for row in history_rows:
+            ds = row.ds
+            final = float(row.final) if row.final else 0
+            prior_ds = ds - timedelta(days=364)
+
+            prior_final = final_by_date.get(prior_ds)
+            if prior_final is None:
+                prior_final = 0  # Allow training even without prior year for revenue metrics
+
+            training_rows.append({
+                'ds': ds,
+                'y': final,
+                'lag_364': float(prior_final) if prior_final else 0
+            })
+
+    if len(training_rows) < 30:
+        logger.warning(f"Insufficient data for XGBoost training: {len(training_rows)} rows")
+        return []
+
+    df = pd.DataFrame(training_rows)
+    df['ds'] = pd.to_datetime(df['ds'])
+
+    # Convert to occupancy if needed
+    if metric == "occupancy" and default_bookable_cap > 0:
+        df["y"] = (df["y"] / default_bookable_cap) * 100
+        if "current_otb" in df.columns:
+            df["current_otb"] = (df["current_otb"] / default_bookable_cap) * 100
+        if "prior_otb_same_lead" in df.columns:
+            df["prior_otb_same_lead"] = (df["prior_otb_same_lead"] / default_bookable_cap) * 100
+        df["lag_364"] = (df["lag_364"] / default_bookable_cap) * 100
+
+    # Create time-based features
+    df['day_of_week'] = df['ds'].dt.dayofweek
+    df['month'] = df['ds'].dt.month
+    df['week_of_year'] = df['ds'].dt.isocalendar().week.astype(int)
+    df['is_weekend'] = (df['day_of_week'] >= 5).astype(int)
+    df['is_special_date'] = df['ds'].dt.date.apply(lambda x: 1 if x in special_date_set else 0)
+
+    df_train = df.dropna()
+
+    if len(df_train) < 30:
+        logger.warning(f"Insufficient data after creating features: {len(df_train)} rows")
+        return []
+
+    # Define features based on metric type
+    if is_room_based:
+        feature_cols = ['day_of_week', 'month', 'week_of_year', 'is_weekend', 'is_special_date',
+                       'days_out', 'current_otb', 'prior_otb_same_lead', 'lag_364', 'otb_pct_of_prior_final']
+    else:
+        feature_cols = ['day_of_week', 'month', 'week_of_year', 'is_weekend', 'is_special_date', 'lag_364']
+
+    X_train = df_train[feature_cols]
+    y_train = df_train['y']
+
+    # Train XGBoost model
+    model = XGBRegressor(
+        n_estimators=100,
+        max_depth=6,
+        learning_rate=0.1,
+        objective='reg:squarederror',
+        random_state=42,
+        n_jobs=-1
+    )
+    model.fit(X_train, y_train)
+
+    # Create future dataframe for forecast period
+    future_dates = []
+    current_date = start_date
+    while current_date <= end_date:
+        if (current_date - today).days >= 0:
+            future_dates.append(current_date)
+        current_date += timedelta(days=1)
+
+    if not future_dates:
+        logger.warning("No future dates to forecast")
+        return []
+
+    # Generate forecasts for each date
+    forecasts = []
+
+    for forecast_date in future_dates:
+        lead_days = (forecast_date - today).days
+        lead_col = get_lead_time_column(lead_days)
+        prior_year_date = forecast_date - timedelta(days=364)
+
+        # Get OTB data only for room-based metrics
+        current_otb = None
+
+        if is_room_based:
+            current_query = text("""
+                SELECT booking_count as current_otb
+                FROM newbook_bookings_stats
+                WHERE date = :arrival_date
+            """)
+            current_result = await db.execute(current_query, {"arrival_date": forecast_date})
+            current_row = current_result.fetchone()
+            current_otb = current_row.current_otb if current_row and current_row.current_otb is not None else 0
+
+        # Get prior year final using metric mapping
+        prior_query = f"""
+            SELECT {col_expr} as prior_final
+            {from_clause}
+            WHERE s.date = :prior_date
+        """
+        prior_result = await db.execute(text(prior_query), {"prior_date": prior_year_date})
+        prior_row = prior_result.fetchone()
+        prior_final = float(prior_row.prior_final) if prior_row and prior_row.prior_final is not None else 0
+
+        # Get per-date bookable cap for this forecast date
+        date_bookable_cap = await get_bookable_cap(db, forecast_date, default_bookable_cap)
+
+        # Build features for this date
+        forecast_dt = pd.Timestamp(forecast_date)
+        lag_364_val = prior_final if prior_final else 0
+
+        # Convert to occupancy if needed
+        if metric == "occupancy" and date_bookable_cap > 0:
+            if current_otb is not None:
+                current_otb = (current_otb / date_bookable_cap) * 100
+            lag_364_val = (prior_final / date_bookable_cap) * 100 if prior_final else 0
+
+        # Build features based on metric type
+        if is_room_based:
+            # Get prior OTB at same lead time
+            prior_year_for_otb = forecast_date - timedelta(days=364)
+            prior_otb_query = text(f"""
+                SELECT {lead_col} as prior_otb
+                FROM newbook_booking_pace
+                WHERE arrival_date = :prior_date
+            """)
+            prior_otb_result = await db.execute(prior_otb_query, {"prior_date": prior_year_for_otb})
+            prior_otb_row = prior_otb_result.fetchone()
+            prior_otb_same_lead = prior_otb_row.prior_otb if prior_otb_row and prior_otb_row.prior_otb is not None else 0
+
+            if metric == "occupancy" and date_bookable_cap > 0:
+                prior_otb_same_lead = (prior_otb_same_lead / date_bookable_cap) * 100 if prior_otb_same_lead else 0
+
+            current_otb_val = current_otb if current_otb is not None else 0
+            otb_pct_of_prior_final = (current_otb_val / lag_364_val * 100) if lag_364_val > 0 else 0
+
+            features = pd.DataFrame([{
+                'day_of_week': forecast_dt.dayofweek,
+                'month': forecast_dt.month,
+                'week_of_year': forecast_dt.isocalendar().week,
+                'is_weekend': 1 if forecast_dt.dayofweek >= 5 else 0,
+                'is_special_date': 1 if forecast_date in special_date_set else 0,
+                'days_out': lead_days,
+                'current_otb': current_otb_val,
+                'prior_otb_same_lead': prior_otb_same_lead,
+                'lag_364': lag_364_val,
+                'otb_pct_of_prior_final': otb_pct_of_prior_final,
+            }])
+        else:
+            features = pd.DataFrame([{
+                'day_of_week': forecast_dt.dayofweek,
+                'month': forecast_dt.month,
+                'week_of_year': forecast_dt.isocalendar().week,
+                'is_weekend': 1 if forecast_dt.dayofweek >= 5 else 0,
+                'is_special_date': 1 if forecast_date in special_date_set else 0,
+                'lag_364': lag_364_val,
+            }])
+
+        # Predict
+        yhat = float(model.predict(features)[0])
+
+        # Cap at max capacity based on metric type (uses per-date bookable cap)
+        if is_pct_metric:
+            yhat = min(max(yhat, 0), 100.0)
+        elif metric == 'rooms':
+            yhat = round(min(max(yhat, 0), float(date_bookable_cap)))
+        elif metric == 'guests':
+            yhat = round(max(yhat, 0))
+        else:
+            # Revenue/rate metrics: just ensure non-negative
+            yhat = max(yhat, 0)
+
+        # Floor forecast to current OTB (room-based only)
+        if is_room_based and current_otb is not None and yhat < current_otb:
+            yhat = current_otb
+
+        # Round based on metric type
+        if metric == "occupancy":
+            yhat = round(yhat, 1)
+        else:
+            yhat = round_towards_reference(yhat, prior_final)
+
+        forecasts.append({
+            'forecast_date': forecast_date,
+            'predicted_value': yhat
+        })
+
+    logger.info(f"XGBoost tuned generated {len(forecasts)} forecasts for {metric_code}")
+    return forecasts

backend/services/newbook_client.py

@@ -0,0 +1,443 @@
+"""
+Newbook API Client
+
+CRITICAL: This client is READ-ONLY.
+Newbook API uses POST for all requests - the "action" parameter determines the operation.
+This client ONLY uses read actions (bookings_list, site_list, report_*).
+NO write actions (booking_create, booking_update, booking_cancel, etc.) are used.
+Data flows ONE WAY: Newbook → Local Database
+"""
+import os
+import httpx
+import asyncio
+import logging
+from datetime import date, timedelta
+from typing import Optional, List
+
+logger = logging.getLogger(__name__)
+
+
+class NewbookAPIError(Exception):
+    """Custom exception for Newbook API errors"""
+    pass
+
+
+class NewbookClient:
+    """
+    Async client for Newbook REST API
+
+    Rate limiting: ~100 requests/min, using 0.75s delay between requests
+    Pagination: Uses data_offset/data_limit, max 1000 per request
+    """
+
+    BASE_URL = "https://api.newbook.cloud/rest"
+
+    def __init__(self, api_key: str = None, username: str = None, password: str = None, region: str = None):
+        # Use provided credentials or fall back to environment variables
+        self.api_key = api_key or os.getenv("NEWBOOK_API_KEY")
+        self.username = username or os.getenv("NEWBOOK_USERNAME")
+        self.password = password or os.getenv("NEWBOOK_PASSWORD")
+        self.region = region or os.getenv("NEWBOOK_REGION")
+
+        if not all([self.api_key, self.username, self.password, self.region]):
+            logger.warning("Newbook credentials not fully configured")
+
+    def _get_url(self, endpoint: str) -> str:
+        """Get full URL for an endpoint"""
+        return f"{self.BASE_URL}/{endpoint}"
+
+    @classmethod
+    async def from_db(cls, db):
+        """Create client with credentials from database"""
+        from api.config import _get_config_value
+
+        api_key = await _get_config_value(db, "newbook_api_key")
+        username = await _get_config_value(db, "newbook_username")
+        password = await _get_config_value(db, "newbook_password")
+        region = await _get_config_value(db, "newbook_region")
+
+        return cls(api_key=api_key, username=username, password=password, region=region)
+
+    async def __aenter__(self):
+        self.client = httpx.AsyncClient(timeout=300.0)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.client.aclose()
+
+    def _get_auth_payload(self) -> dict:
+        """Get base authentication payload (api_key and region only - username/password go in Basic Auth)"""
+        return {
+            "api_key": self.api_key,
+            "region": self.region
+        }
+
+    async def test_connection(self) -> bool:
+        """Test API connection"""
+        try:
+            payload = self._get_auth_payload()
+
+            response = await self.client.post(
+                self._get_url("site_list"),
+                json=payload,
+                auth=(self.username, self.password)
+            )
+            return response.status_code == 200
+        except Exception as e:
+            logger.error(f"Newbook connection test failed: {e}")
+            return False
+
+    async def get_bookings(
+        self,
+        modified_since: Optional[str] = None,
+        modified_until: Optional[str] = None,
+        batch_size: int = 1000
+    ) -> List[dict]:
+        """
+        Fetch all bookings with pagination and rate limiting.
+
+        Uses list_type="all" which returns all bookings (including cancelled).
+        period_from/period_to filter by created/modified timestamp, not stay dates.
+
+        Args:
+            modified_since: ISO timestamp - only bookings created/modified after this
+            modified_until: ISO timestamp - only bookings created/modified before this
+            batch_size: Records per request (max 1000)
+
+        Returns:
+            List of booking objects (all statuses including cancelled)
+        """
+        all_bookings = []
+        offset = 0
+
+        while True:
+            logger.info(f"Fetching Newbook bookings (all): modified_since={modified_since} (offset: {offset})")
+
+            payload = self._get_auth_payload()
+            payload.update({
+                "list_type": "all",
+                "data_offset": offset,
+                "data_limit": batch_size
+            })
+
+            # Add optional timestamp filters
+            if modified_since:
+                payload["period_from"] = modified_since
+            if modified_until:
+                payload["period_to"] = modified_until
+
+            response = await self.client.post(
+                self._get_url("bookings_list"),
+                json=payload,
+                auth=(self.username, self.password)
+            )
+
+            if response.status_code != 200:
+                logger.error(f"Newbook API error {response.status_code}: {response.text}")
+                raise NewbookAPIError(f"Failed to fetch bookings: {response.status_code}")
+
+            data = response.json()
+
+            if not data.get("success"):
+                raise NewbookAPIError(f"Newbook API returned failure: {data.get('message')}")
+
+            bookings = data.get("data", [])
+
+            if not bookings:
+                break
+
+            all_bookings.extend(bookings)
+            logger.info(f"Fetched {len(bookings)} bookings (offset {offset})")
+
+            # Check if we've got all records
+            total = data.get("data_total", 0)
+            if offset + len(bookings) >= total:
+                break
+
+            offset += batch_size
+
+            # Rate limiting: 0.75s delay
+            await asyncio.sleep(0.75)
+
+        logger.info(f"Total bookings fetched: {len(all_bookings)}")
+        return all_bookings
+
+    async def get_bookings_by_stay_dates(
+        self,
+        from_date: date,
+        to_date: date,
+        list_type: str = "staying",
+        batch_size: int = 1000
+    ) -> List[dict]:
+        """
+        Fetch bookings by stay dates (arrival/departure/staying period).
+
+        Args:
+            from_date: Start date for stay period
+            to_date: End date for stay period
+            list_type: Type of booking list:
+                       "staying" - bookings staying during dates (excludes cancelled)
+                       "arrived" - arrived during dates (add mode="projected" for expected)
+                       "arriving" - expected to arrive before period_to
+                       "departed" - departed during dates
+                       "departing" - expected to depart during dates
+                       "cancelled" - cancelled during dates
+                       "placed" - created during dates
+                       "no_show" - no shows for dates
+            batch_size: Records per request (max 1000)
+
+        Returns:
+            List of booking objects
+        """
+        all_bookings = []
+        offset = 0
+
+        while True:
+            logger.info(f"Fetching Newbook bookings ({list_type}): {from_date} to {to_date} (offset: {offset})")
+
+            payload = self._get_auth_payload()
+            payload.update({
+                "list_type": list_type,
+                "period_from": from_date.isoformat(),
+                "period_to": to_date.isoformat(),
+                "data_offset": offset,
+                "data_limit": batch_size
+            })
+
+            response = await self.client.post(
+                self._get_url("bookings_list"),
+                json=payload,
+                auth=(self.username, self.password)
+            )
+
+            if response.status_code != 200:
+                logger.error(f"Newbook API error {response.status_code}: {response.text}")
+                raise NewbookAPIError(f"Failed to fetch bookings: {response.status_code}")
+
+            data = response.json()
+
+            if not data.get("success"):
+                raise NewbookAPIError(f"Newbook API returned failure: {data.get('message')}")
+
+            bookings = data.get("data", [])
+
+            if not bookings:
+                break
+
+            all_bookings.extend(bookings)
+            logger.info(f"Fetched {len(bookings)} bookings (offset {offset})")
+
+            # Check if we've got all records
+            total = data.get("data_total", 0)
+            if offset + len(bookings) >= total:
+                break
+
+            offset += batch_size
+
+            # Rate limiting: 0.75s delay
+            await asyncio.sleep(0.75)
+
+        logger.info(f"Total bookings fetched: {len(all_bookings)}")
+        return all_bookings
+
+    async def get_occupancy_report(
+        self,
+        from_date: date,
+        to_date: date
+    ) -> List[dict]:
+        """
+        Fetch occupancy report for date range.
+
+        Uses reports_occupancy endpoint which returns data by room category.
+        Returns all categories with nested occupancy data for each date in range.
+        No pagination needed - API returns full dataset in single response.
+
+        Response format:
+        [
+            {
+                "category_id": "1",
+                "category_name": "Single Room",
+                "occupancy": {
+                    "2024-08-01": {
+                        "date": "2024-08-01",
+                        "available": 5,
+                        "occupied": 3,
+                        "maintenance": 1,
+                        "allotted": 0,
+                        "revenue_gross": 450.00,
+                        "revenue_net": 375.00
+                    },
+                    ...
+                }
+            },
+            ...
+        ]
+
+        Returns list of category objects with nested occupancy by date
+        """
+        logger.info(f"Fetching occupancy report: {from_date} to {to_date}")
+
+        payload = self._get_auth_payload()
+        payload.update({
+            "period_from": f"{from_date.isoformat()} 00:00:00",
+            "period_to": f"{to_date.isoformat()} 23:59:59"
+        })
+
+        response = await self.client.post(
+            self._get_url("reports_occupancy"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        if response.status_code != 200:
+            raise NewbookAPIError(f"Failed to fetch occupancy: {response.status_code}")
+
+        data = response.json()
+
+        if not data.get("success"):
+            raise NewbookAPIError(f"Newbook API returned failure: {data.get('message')}")
+
+        records = data.get("data", [])
+        logger.info(f"Fetched occupancy report: {len(records)} categories")
+        return records
+
+    async def get_site_list(self) -> List[dict]:
+        """Fetch list of rooms/sites with categories"""
+        payload = self._get_auth_payload()
+
+        response = await self.client.post(
+            self._get_url("site_list"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        if response.status_code != 200:
+            raise NewbookAPIError(f"Failed to fetch site list: {response.status_code}")
+
+        data = response.json()
+        return data.get("data", [])
+
+    async def get_earned_revenue(
+        self,
+        from_date: date,
+        to_date: date
+    ) -> dict:
+        """
+        Fetch earned revenue report day by day
+
+        Returns dict keyed by date with revenue breakdown by GL code
+        """
+        all_revenue = {}
+        current_date = from_date
+
+        while current_date <= to_date:
+            logger.info(f"Fetching earned revenue for {current_date}")
+
+            payload = self._get_auth_payload()
+            payload.update({
+                "period_from": current_date.isoformat(),
+                "period_to": current_date.isoformat()
+            })
+
+            response = await self.client.post(
+                self._get_url("reports_earned_revenue"),
+                json=payload,
+                auth=(self.username, self.password)
+            )
+
+            if response.status_code == 200:
+                data = response.json()
+                if data.get("success"):
+                    day_data = data.get("data", {})
+                    # Debug: log first day's response structure
+                    if current_date == from_date:
+                        import json
+                        logger.info(f"Sample earned revenue response: {json.dumps(day_data)[:500]}")
+                    all_revenue[current_date.isoformat()] = day_data
+
+            current_date += timedelta(days=1)
+
+            # Rate limiting
+            await asyncio.sleep(0.75)
+
+        return all_revenue
+
+    async def get_transaction_flow(
+        self,
+        from_date: date,
+        to_date: date,
+        batch_size: int = 5000
+    ) -> List[dict]:
+        """
+        Fetch transaction flow report for date range.
+        Used by reconciliation module for payment categorization.
+
+        Returns raw transaction records (payments, refunds, voided items).
+        Excludes balance_transfer items.
+        Handles pagination via data_offset/data_limit.
+        """
+        logger.info(f"Fetching transaction flow: {from_date} to {to_date}")
+
+        all_transactions = []
+        offset = 0
+
+        while True:
+            payload = self._get_auth_payload()
+            payload.update({
+                "period_from": f"{from_date.isoformat()} 00:00:00",
+                "period_to": f"{to_date.isoformat()} 23:59:59",
+                "data_offset": offset,
+                "data_limit": batch_size
+            })
+
+            response = await self.client.post(
+                self._get_url("reports_transaction_flow"),
+                json=payload,
+                auth=(self.username, self.password)
+            )
+
+            if response.status_code != 200:
+                raise NewbookAPIError(f"Failed to fetch transaction flow: {response.status_code}")
+
+            data = response.json()
+            if not data.get("success"):
+                raise NewbookAPIError(f"Newbook API returned failure: {data.get('message')}")
+
+            records = data.get("data", [])
+            all_transactions.extend(records)
+
+            # If we got fewer records than the limit, we're done
+            if len(records) < batch_size:
+                break
+
+            offset += batch_size
+            await asyncio.sleep(0.75)
+
+        logger.info(f"Fetched transaction flow: {len(all_transactions)} transactions")
+        return all_transactions
+
+    async def get_gl_account_list(self) -> List[dict]:
+        """
+        Fetch GL account list from Newbook.
+        Used for reconciliation sales breakdown column configuration.
+        """
+        logger.info("Fetching GL account list from Newbook")
+
+        payload = self._get_auth_payload()
+
+        response = await self.client.post(
+            self._get_url("gl_account_list"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        if response.status_code != 200:
+            raise NewbookAPIError(f"Failed to fetch GL accounts: {response.status_code}")
+
+        data = response.json()
+        if not data.get("success"):
+            raise NewbookAPIError(f"Newbook API returned failure: {data.get('message')}")
+
+        records = data.get("data", [])
+        logger.info(f"Fetched {len(records)} GL accounts")
+        return records

backend/services/newbook_rates_client.py

@@ -0,0 +1,855 @@
+"""
+Newbook Rates Client
+
+Fetches current rack rates from Newbook API for revenue forecasting.
+Uses the bookings_availability_pricing endpoint to simulate booking requests.
+
+This client is READ-ONLY - it only queries available rates, never creates bookings.
+"""
+import os
+import httpx
+import asyncio
+import logging
+from datetime import date, timedelta
+from decimal import Decimal
+from typing import Optional, List, Dict
+
+logger = logging.getLogger(__name__)
+
+
+class NewbookRatesError(Exception):
+    """Custom exception for Newbook rates API errors"""
+    pass
+
+
+class NewbookRatesClient:
+    """
+    Async client for fetching current rates from Newbook API.
+
+    Uses bookings_availability_pricing endpoint which simulates a booking request.
+    Handles minimum stay restrictions by extending the stay period when needed.
+
+    Rate limiting: ~100 requests/min, using 0.75s delay between requests
+    """
+
+    BASE_URL = "https://api.newbook.cloud/rest"
+
+    def __init__(self, api_key: str = None, username: str = None, password: str = None,
+                 region: str = None, vat_rate: Decimal = Decimal('0.20')):
+        self.api_key = api_key or os.getenv("NEWBOOK_API_KEY")
+        self.username = username or os.getenv("NEWBOOK_USERNAME")
+        self.password = password or os.getenv("NEWBOOK_PASSWORD")
+        self.region = region or os.getenv("NEWBOOK_REGION")
+        self.vat_rate = vat_rate
+
+        if not all([self.api_key, self.username, self.password, self.region]):
+            logger.warning("Newbook credentials not fully configured")
+
+    def _get_url(self, endpoint: str) -> str:
+        """Get full URL for an endpoint"""
+        return f"{self.BASE_URL}/{endpoint}"
+
+    @classmethod
+    async def from_db(cls, db):
+        """Create client with credentials and VAT rate from database"""
+        from sqlalchemy import text
+
+        # Get credentials from config
+        result = await db.execute(
+            text("SELECT config_key, config_value FROM system_config WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region', 'accommodation_vat_rate')")
+        )
+        rows = result.fetchall()
+        config = {row.config_key: row.config_value for row in rows}
+
+        vat_rate = Decimal(config.get('accommodation_vat_rate', '0.20'))
+
+        return cls(
+            api_key=config.get('newbook_api_key'),
+            username=config.get('newbook_username'),
+            password=config.get('newbook_password'),
+            region=config.get('newbook_region'),
+            vat_rate=vat_rate
+        )
+
+    async def __aenter__(self):
+        self.client = httpx.AsyncClient(timeout=60.0)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.client.aclose()
+
+    def _get_auth_payload(self) -> dict:
+        """Get base authentication payload"""
+        return {
+            "api_key": self.api_key,
+            "region": self.region
+        }
+
+    async def get_category_rates(
+        self,
+        category_id: str,
+        from_date: date,
+        to_date: date,
+        guests_adults: int = 2,
+        guests_children: int = 0
+    ) -> List[Dict]:
+        """
+        Fetch current rates for a category over a date range.
+
+        Uses daily=true to get per-night rates. Handles minimum stay
+        restrictions by extending the period when needed.
+
+        Args:
+            category_id: Newbook category ID
+            from_date: Start date for rates
+            to_date: End date for rates (inclusive)
+            guests_adults: Number of adult guests (default 2)
+            guests_children: Number of child guests (default 0)
+
+        Returns:
+            List of dicts with {date, gross_rate, net_rate}
+        """
+        rates = []
+        current_date = from_date
+
+        while current_date <= to_date:
+            try:
+                # Fetch rates for up to 7 days at a time to optimize API calls
+                batch_end = min(current_date + timedelta(days=6), to_date)
+                batch_rates = await self._fetch_rates_batch(
+                    category_id, current_date, batch_end, guests_adults, guests_children
+                )
+                rates.extend(batch_rates)
+
+                # Move to next batch
+                current_date = batch_end + timedelta(days=1)
+
+            except Exception as e:
+                logger.error(f"Failed to fetch rates for category {category_id} starting {current_date}: {e}")
+                # Skip this batch and continue
+                current_date = current_date + timedelta(days=7)
+
+            # Rate limiting - ALWAYS wait 1.5s between requests, even after errors
+            await asyncio.sleep(1.5)
+
+        return rates
+
+    async def get_single_night_rates(
+        self,
+        category_id: str,
+        from_date: date,
+        to_date: date,
+        guests_adults: int = 2,
+        guests_children: int = 0
+    ) -> List[Dict]:
+        """
+        Fetch rates with single-night queries for accurate per-day tariff availability.
+
+        Unlike get_category_rates which batches, this queries each date individually
+        as a 1-night stay. This gives accurate tariff_success per night, catching
+        issues like Valentine's Day blocking only that night, not a whole week.
+
+        Much slower but necessary for accurate bookability data.
+
+        Args:
+            category_id: Newbook category ID
+            from_date: Start date for rates
+            to_date: End date for rates (inclusive)
+            guests_adults: Number of adult guests (default 2)
+            guests_children: Number of child guests (default 0)
+
+        Returns:
+            List of dicts with {date, gross_rate, net_rate, tariffs_data}
+        """
+        rates = []
+        current_date = from_date
+
+        while current_date <= to_date:
+            try:
+                # Single-night query for accurate tariff availability
+                batch_rates = await self._fetch_rates_batch(
+                    category_id, current_date, current_date, guests_adults, guests_children
+                )
+                rates.extend(batch_rates)
+
+            except Exception as e:
+                logger.warning(f"Failed to fetch single-night rate for {category_id} on {current_date}: {e}")
+                # Continue with next date
+
+            current_date += timedelta(days=1)
+
+            # Rate limiting - wait between each single-night query
+            await asyncio.sleep(1.0)
+
+        return rates
+
+    async def fetch_single_date_all_categories(
+        self,
+        for_date: date,
+        guests_adults: int = 2,
+        guests_children: int = 0
+    ) -> Dict[str, List[Dict]]:
+        """
+        Fetch single-night rates for ALL categories for one date.
+
+        Returns:
+            Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
+        """
+        return await self._fetch_all_categories_batch(
+            for_date, guests_adults, guests_children
+        )
+
+    async def fetch_multi_night_for_date(
+        self,
+        for_date: date,
+        nights: int,
+        guests_adults: int = 2,
+        guests_children: int = 0
+    ) -> Dict[str, Dict[str, bool]]:
+        """
+        Fetch multi-night availability for ALL categories for one date.
+
+        Returns:
+            Dict of {category_id: {tariff_name: available}}
+        """
+        return await self._fetch_all_categories_multi_night(
+            for_date, nights, guests_adults, guests_children
+        )
+
+    async def get_all_categories_single_night_rates(
+        self,
+        from_date: date,
+        to_date: date,
+        guests_adults: int = 2,
+        guests_children: int = 0
+    ) -> Dict[str, List[Dict]]:
+        """
+        Fetch rates for ALL categories with single-night queries.
+
+        More efficient than get_single_night_rates - omits category_id to get
+        all categories in a single API call per date. This reduces API calls
+        from (categories × days) to just (days).
+
+        Args:
+            from_date: Start date for rates
+            to_date: End date for rates (inclusive)
+            guests_adults: Number of adult guests (default 2)
+            guests_children: Number of child guests (default 0)
+
+        Returns:
+            Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}, ...]}
+        """
+        all_rates: Dict[str, List[Dict]] = {}
+        current_date = from_date
+        total_days = (to_date - from_date).days + 1
+        day_count = 0
+
+        while current_date <= to_date:
+            day_count += 1
+            try:
+                # Single-night query WITHOUT category_id - returns ALL categories
+                category_rates = await self._fetch_all_categories_batch(
+                    current_date, guests_adults, guests_children
+                )
+
+                # Merge into all_rates dict
+                for cat_id, rates in category_rates.items():
+                    if cat_id not in all_rates:
+                        all_rates[cat_id] = []
+                    all_rates[cat_id].extend(rates)
+
+                logger.info(f"Fetched {current_date} ({day_count}/{total_days}) - {len(category_rates)} categories")
+
+            except Exception as e:
+                logger.warning(f"Failed to fetch rates for {current_date}: {e}")
+                # Continue with next date
+
+            current_date += timedelta(days=1)
+
+            # Rate limiting - wait between each query
+            await asyncio.sleep(1.0)
+
+        return all_rates
+
+    async def _fetch_all_categories_batch(
+        self,
+        for_date: date,
+        guests_adults: int,
+        guests_children: int,
+        retry_count: int = 0
+    ) -> Dict[str, List[Dict]]:
+        """
+        Fetch rates for ALL categories for a single date.
+
+        Omits category_id from request - Newbook returns all available categories.
+
+        Returns:
+            Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
+        """
+        # Single-night query
+        period_from = f"{for_date.isoformat()} 14:00:00"
+        period_to = f"{(for_date + timedelta(days=1)).isoformat()} 10:00:00"
+
+        payload = self._get_auth_payload()
+        payload.update({
+            "period_from": period_from,
+            "period_to": period_to,
+            "adults": guests_adults,
+            "children": guests_children,
+            "infants": 0,
+            "daily_mode": "true"
+            # NO category_id - returns all categories
+        })
+
+        response = await self.client.post(
+            self._get_url("bookings_availability_pricing"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        # Handle rate limiting with exponential backoff
+        if response.status_code == 429:
+            if retry_count < 3:
+                wait_time = 60 * (retry_count + 1)
+                logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3")
+                await asyncio.sleep(wait_time)
+                return await self._fetch_all_categories_batch(
+                    for_date, guests_adults, guests_children, retry_count + 1
+                )
+            else:
+                raise NewbookRatesError(f"Rate limited after 3 retries")
+
+        if response.status_code != 200:
+            raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
+
+        data = response.json()
+
+        if not data.get("success"):
+            raise NewbookRatesError(f"API returned failure: {data.get('message')}")
+
+        # Parse all categories from response
+        return self._parse_all_categories_tariffs(data, for_date)
+
+    async def _fetch_all_categories_multi_night(
+        self,
+        for_date: date,
+        nights: int,
+        guests_adults: int = 2,
+        guests_children: int = 0,
+        retry_count: int = 0
+    ) -> Dict[str, Dict[str, bool]]:
+        """
+        Fetch multi-night availability for ALL categories for a specific date.
+
+        Used to verify that rates with min_stay requirements are actually bookable.
+
+        Args:
+            for_date: Check-in date
+            nights: Number of nights to query (e.g., 2 for min_stay=2)
+            guests_adults: Number of adult guests
+            guests_children: Number of child guests
+
+        Returns:
+            Dict of {category_id: {tariff_name: available}}
+        """
+        # Multi-night query
+        period_from = f"{for_date.isoformat()} 14:00:00"
+        period_to = f"{(for_date + timedelta(days=nights)).isoformat()} 10:00:00"
+
+        payload = self._get_auth_payload()
+        payload.update({
+            "period_from": period_from,
+            "period_to": period_to,
+            "adults": guests_adults,
+            "children": guests_children,
+            "infants": 0,
+            "daily_mode": "true"
+        })
+
+        response = await self.client.post(
+            self._get_url("bookings_availability_pricing"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        # Handle rate limiting
+        if response.status_code == 429:
+            if retry_count < 3:
+                wait_time = 60 * (retry_count + 1)
+                logger.warning(f"Rate limited (multi-night), waiting {wait_time}s")
+                await asyncio.sleep(wait_time)
+                return await self._fetch_all_categories_multi_night(
+                    for_date, nights, guests_adults, guests_children, retry_count + 1
+                )
+            else:
+                raise NewbookRatesError(f"Rate limited after 3 retries")
+
+        if response.status_code != 200:
+            raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
+
+        data = response.json()
+
+        if not data.get("success"):
+            raise NewbookRatesError(f"API returned failure: {data.get('message')}")
+
+        # Parse availability by tariff name for each category
+        results: Dict[str, Dict[str, bool]] = {}
+
+        if not isinstance(data.get("data"), dict):
+            return results
+
+        for key, cat_data in data["data"].items():
+            if not (key.isdigit() or str(key).isnumeric()):
+                continue
+            if not isinstance(cat_data, dict):
+                continue
+
+            category_id = str(key)
+            tariffs_available = cat_data.get("tariffs_available", [])
+
+            results[category_id] = {}
+            for tariff in tariffs_available:
+                tariff_name = tariff.get("tariff_name", "")
+                tariff_label = tariff.get("tariff_label", "")
+                # Check tariff_success (API returns string "true"/"false")
+                tariff_success = str(tariff.get("tariff_success", False)).lower() in ("true", "1")
+                # Available if API says success, OR if rates are quoted and no restriction message
+                is_available = tariff_success or (
+                    bool(tariff.get("tariffs_quoted")) and not tariff.get("tariff_message")
+                )
+                # Store under both tariff_name and tariff_label for flexible matching
+                results[category_id][tariff_name] = is_available
+                if tariff_label and tariff_label != tariff_name:
+                    results[category_id][tariff_label] = is_available
+
+        return results
+
+    async def get_multi_night_availability(
+        self,
+        dates_by_nights: Dict[int, List[date]],
+        guests_adults: int = 2,
+        guests_children: int = 0
+    ) -> Dict[date, Dict[str, Dict[str, bool]]]:
+        """
+        Fetch multi-night availability for specific dates grouped by stay length.
+
+        Checks if a tariff is available when booking N nights starting from each date.
+
+        Args:
+            dates_by_nights: Dict of {nights: [dates]} e.g., {2: [date1, date2], 3: [date3]}
+            guests_adults: Number of adult guests
+            guests_children: Number of child guests
+
+        Returns:
+            Dict of {date: {category_id: {tariff_name: available}}}
+        """
+        results: Dict[date, Dict[str, Dict[str, bool]]] = {}
+
+        total_queries = sum(len(dates) for dates in dates_by_nights.values())
+        query_count = 0
+
+        for nights, dates in dates_by_nights.items():
+            for query_date in dates:
+                query_count += 1
+
+                try:
+                    result = await self._fetch_all_categories_multi_night(
+                        query_date, nights, guests_adults, guests_children
+                    )
+                    results[query_date] = result
+                    logger.info(f"Multi-night check {query_count}/{total_queries}: {query_date} ({nights} nights)")
+                except Exception as e:
+                    logger.warning(f"Failed multi-night check for {query_date}: {e}")
+
+                # Rate limiting
+                await asyncio.sleep(1.0)
+
+        return results
+
+    async def _fetch_rates_batch(
+        self,
+        category_id: str,
+        from_date: date,
+        to_date: date,
+        guests_adults: int,
+        guests_children: int,
+        retry_count: int = 0
+    ) -> List[Dict]:
+        """
+        Fetch rates for a batch of dates (up to 7 days).
+
+        Handles minimum stay restrictions by extending the period and
+        extracting only the dates we need.
+
+        Returns:
+            List of dicts with {date, gross_rate, net_rate}
+        """
+        # Format dates with times (check-in 14:00, check-out 10:00)
+        period_from = f"{from_date.isoformat()} 14:00:00"
+        period_to = f"{(to_date + timedelta(days=1)).isoformat()} 10:00:00"
+
+        payload = self._get_auth_payload()
+        payload.update({
+            "period_from": period_from,
+            "period_to": period_to,
+            "adults": guests_adults,
+            "children": guests_children,
+            "infants": 0,
+            "category_id": category_id,
+            "daily_mode": "true"  # Get per-night breakdown
+        })
+
+        response = await self.client.post(
+            self._get_url("bookings_availability_pricing"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        # Handle rate limiting with exponential backoff
+        if response.status_code == 429:
+            if retry_count < 3:
+                wait_time = 60 * (retry_count + 1)  # 60s, 120s, 180s
+                logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry {retry_count + 1}/3")
+                await asyncio.sleep(wait_time)
+                return await self._fetch_rates_batch(
+                    category_id, from_date, to_date, guests_adults, guests_children, retry_count + 1
+                )
+            else:
+                raise NewbookRatesError(f"Rate limited after 3 retries")
+
+        if response.status_code != 200:
+            raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
+
+        data = response.json()
+
+        if not data.get("success"):
+            # Check if minimum stay restriction
+            categories = data.get("data", {}).get("categories", [])
+            if categories:
+                cat = categories[0] if isinstance(categories, list) else categories.get(category_id, {})
+                min_periods = cat.get("minimum_periods", 1)
+
+                if min_periods > 1:
+                    # Extend the stay to meet minimum and retry
+                    extended_to = from_date + timedelta(days=min_periods)
+                    logger.info(f"Minimum stay {min_periods} nights for category {category_id}, extending to {extended_to}")
+                    return await self._fetch_rates_with_min_stay(
+                        category_id, from_date, to_date, extended_to,
+                        guests_adults, guests_children
+                    )
+
+            raise NewbookRatesError(f"API returned failure: {data.get('message')}")
+
+        # Parse tariffs_quoted from response
+        return self._parse_tariffs(data, from_date, to_date)
+
+    async def _fetch_rates_with_min_stay(
+        self,
+        category_id: str,
+        from_date: date,
+        to_date: date,
+        extended_to: date,
+        guests_adults: int,
+        guests_children: int,
+        retry_count: int = 0
+    ) -> List[Dict]:
+        """
+        Fetch rates with extended period for minimum stay requirement.
+
+        Args:
+            category_id: Newbook category ID
+            from_date: Original start date
+            to_date: Original end date (dates we want)
+            extended_to: Extended end date to meet minimum stay
+            guests_adults: Number of adults
+            guests_children: Number of children
+
+        Returns:
+            List of rates for the original date range only
+        """
+        period_from = f"{from_date.isoformat()} 14:00:00"
+        period_to = f"{(extended_to + timedelta(days=1)).isoformat()} 10:00:00"
+
+        payload = self._get_auth_payload()
+        payload.update({
+            "period_from": period_from,
+            "period_to": period_to,
+            "adults": guests_adults,
+            "children": guests_children,
+            "infants": 0,
+            "category_id": category_id,
+            "daily_mode": "true"
+        })
+
+        response = await self.client.post(
+            self._get_url("bookings_availability_pricing"),
+            json=payload,
+            auth=(self.username, self.password)
+        )
+
+        # Handle rate limiting with exponential backoff
+        if response.status_code == 429:
+            if retry_count < 3:
+                wait_time = 60 * (retry_count + 1)
+                logger.warning(f"Rate limited by Newbook API, waiting {wait_time}s before retry")
+                await asyncio.sleep(wait_time)
+                return await self._fetch_rates_with_min_stay(
+                    category_id, from_date, to_date, extended_to, guests_adults, guests_children, retry_count + 1
+                )
+            else:
+                raise NewbookRatesError(f"Rate limited after 3 retries")
+
+        if response.status_code != 200:
+            raise NewbookRatesError(f"API error {response.status_code}: {response.text}")
+
+        data = response.json()
+
+        if not data.get("success"):
+            raise NewbookRatesError(f"API returned failure even with extended stay: {data.get('message')}")
+
+        # Parse tariffs but only return dates in our original range
+        return self._parse_tariffs(data, from_date, to_date)
+
+    def _parse_tariffs(self, data: dict, from_date: date, to_date: date) -> List[Dict]:
+        """
+        Parse tariffs from API response.
+
+        With daily_mode=true, the API returns tariffs_quoted as a dict keyed by date.
+        Falls back to tariffs_available average if tariffs_quoted not available.
+
+        Args:
+            data: Full API response
+            from_date: Start date to include
+            to_date: End date to include
+
+        Returns:
+            List of dicts with {date, gross_rate, net_rate, tariffs_data}
+            tariffs_data contains all available tariff options for rate report
+        """
+        rates = []
+        tariffs_quoted = {}
+        fallback_rate = None
+        inventory_items = []
+        all_tariffs_available = []  # Store all tariff options for reporting
+
+        # Find tariffs data in the response
+        if isinstance(data.get("data"), dict):
+            for key in data["data"].keys():
+                # Category IDs are numeric strings
+                if key.isdigit() or key.isnumeric():
+                    cat_data = data["data"][key]
+                    if isinstance(cat_data, dict):
+                        tariffs_available = cat_data.get("tariffs_available", [])
+                        all_tariffs_available = tariffs_available  # Capture all options
+                        if tariffs_available:
+                            first_tariff = tariffs_available[0]
+                            # tariffs_quoted is a dict keyed by date string
+                            tariffs_quoted = first_tariff.get("tariffs_quoted", {})
+                            # inventory_items are at tariff level (total for whole stay)
+                            inventory_items = first_tariff.get("inventory_items", [])
+                            # Fallback average rate
+                            fallback_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0))
+                        break
+
+        # If we have per-night tariffs_quoted dict, parse it
+        if isinstance(tariffs_quoted, dict) and tariffs_quoted:
+            num_nights = len(tariffs_quoted)
+
+            # Calculate per-night inventory item amount for items already included in tariff
+            included_inventory_per_night = Decimal('0')
+            for item in inventory_items:
+                already_included = item.get('amount_already_included_in_tariff_total', '')
+                if str(already_included).lower() == 'true':
+                    total_amount = Decimal(str(item.get('amount', 0) or 0))
+                    included_inventory_per_night += total_amount / num_nights
+
+            for date_str, tariff in tariffs_quoted.items():
+                try:
+                    stay_date = date.fromisoformat(date_str)
+                except ValueError:
+                    continue
+
+                # Only include dates in our range
+                if stay_date < from_date or stay_date > to_date:
+                    continue
+
+                gross_rate = Decimal(str(tariff.get('amount', 0) or 0))
+                # Net = (gross - included_inventory_per_night) / (1 + VAT)
+                gross_after_inventory = gross_rate - included_inventory_per_night
+                net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01'))
+
+                # Build tariffs_data with day-specific rates
+                tariffs_data = self._build_tariffs_summary(all_tariffs_available, stay_date)
+
+                rates.append({
+                    'date': stay_date,
+                    'gross_rate': float(gross_rate),
+                    'net_rate': float(net_rate),
+                    'tariffs_data': tariffs_data
+                })
+
+            return rates
+
+        # Fallback: use average_nightly_tariff and apply to all dates
+        if fallback_rate and fallback_rate > 0:
+            net_rate = (fallback_rate / (1 + self.vat_rate)).quantize(Decimal('0.01'))
+            current_date = from_date
+            while current_date <= to_date:
+                # Build tariffs_data (no day-specific rates in fallback)
+                tariffs_data = self._build_tariffs_summary(all_tariffs_available, current_date)
+                rates.append({
+                    'date': current_date,
+                    'gross_rate': float(fallback_rate),
+                    'net_rate': float(net_rate),
+                    'tariffs_data': tariffs_data
+                })
+                current_date += timedelta(days=1)
+            return rates
+
+        logger.warning(f"No rate found in response for {from_date} to {to_date}")
+        return rates
+
+    def _build_tariffs_summary(self, tariffs_available: list, for_date: date = None) -> dict:
+        """
+        Build a summary of all available tariff options for rate reporting.
+
+        Args:
+            tariffs_available: List of tariff dicts from API response
+            for_date: Optional specific date to extract day-specific rates
+
+        Returns:
+            Dict with tariff summaries - tariff_count and list of tariff details
+        """
+        if not tariffs_available:
+            return {}
+
+        summary = {
+            'tariff_count': len(tariffs_available),
+            'tariffs': []
+        }
+
+        date_key = for_date.isoformat() if for_date else None
+
+        for idx, tariff in enumerate(tariffs_available):
+            # Get day-specific rate from tariffs_quoted if available
+            day_rate = None
+            if date_key:
+                tariffs_quoted = tariff.get('tariffs_quoted', {})
+                if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted:
+                    day_quote = tariffs_quoted[date_key]
+                    if isinstance(day_quote, dict):
+                        day_rate = float(day_quote.get('amount', 0) or 0)
+                    else:
+                        day_rate = float(day_quote or 0)
+
+            # API uses tariff_label for the name
+            message = tariff.get('tariff_message', '')
+
+            # Extract minimum stay from message or dedicated field
+            min_stay = tariff.get('minimum_nights', None)
+            if min_stay is None and message:
+                # Try to parse from message like "Minimum 2 nights" or "2 Night Minimum"
+                import re
+                match = re.search(r'(\d+)\s*[Nn]ight\s*[Mm]inimum', message)
+                if not match:
+                    match = re.search(r'[Mm]inimum\s+(\d+)\s*(?:night|period)', message)
+                if match:
+                    min_stay = int(match.group(1))
+
+            # Extract advance booking requirement from message
+            min_advance_days = None
+            if message:
+                import re
+                advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
+                if advance_match:
+                    min_advance_days = int(advance_match.group(1))
+
+            tariff_info = {
+                'name': tariff.get('tariff_label', 'Unknown'),
+                'description': tariff.get('tariff_short_description', ''),
+                'rate': day_rate,  # Day-specific rate (None if not available)
+                'average_nightly': float(tariff.get('average_nightly_tariff', 0) or 0),
+                'success': str(tariff.get('tariff_success', False)).lower() in ('true', '1'),
+                'message': message,
+                'sort_order': idx,  # Preserve Newbook ordering
+                'min_stay': min_stay,  # Minimum nights required (if any)
+                'min_advance_days': min_advance_days,  # Advance booking requirement (if any)
+            }
+
+            summary['tariffs'].append(tariff_info)
+
+        return summary
+
+    def _parse_all_categories_tariffs(self, data: dict, for_date: date) -> Dict[str, List[Dict]]:
+        """
+        Parse tariffs from API response for ALL categories.
+
+        When category_id is omitted, data.data contains category IDs as keys,
+        each with their own tariffs_available.
+
+        Args:
+            data: Full API response
+            for_date: The date we queried
+
+        Returns:
+            Dict of {category_id: [{date, gross_rate, net_rate, tariffs_data}]}
+        """
+        results: Dict[str, List[Dict]] = {}
+
+        if not isinstance(data.get("data"), dict):
+            return results
+
+        for key, cat_data in data["data"].items():
+            # Category IDs are numeric strings like "1", "8", etc.
+            if not (key.isdigit() or str(key).isnumeric()):
+                continue
+
+            if not isinstance(cat_data, dict):
+                continue
+
+            category_id = str(key)
+            tariffs_available = cat_data.get("tariffs_available", [])
+
+            if not tariffs_available:
+                continue
+
+            # Get the first (best) tariff for gross/net calculation
+            first_tariff = tariffs_available[0]
+            tariffs_quoted = first_tariff.get("tariffs_quoted", {})
+            inventory_items = first_tariff.get("inventory_items", [])
+
+            # Get rate for this date
+            date_key = for_date.isoformat()
+            gross_rate = Decimal('0')
+            net_rate = Decimal('0')
+
+            if isinstance(tariffs_quoted, dict) and date_key in tariffs_quoted:
+                day_tariff = tariffs_quoted[date_key]
+                gross_rate = Decimal(str(day_tariff.get('amount', 0) or 0))
+
+                # Calculate included inventory per night
+                included_inventory = Decimal('0')
+                for item in inventory_items:
+                    already_included = item.get('amount_already_included_in_tariff_total', '')
+                    if str(already_included).lower() == 'true':
+                        included_inventory += Decimal(str(item.get('amount', 0) or 0))
+
+                gross_after_inventory = gross_rate - included_inventory
+                net_rate = (gross_after_inventory / (1 + self.vat_rate)).quantize(Decimal('0.01'))
+            else:
+                # Fallback to average
+                gross_rate = Decimal(str(first_tariff.get('average_nightly_tariff', 0) or 0))
+                net_rate = (gross_rate / (1 + self.vat_rate)).quantize(Decimal('0.01'))
+
+            # Build tariffs summary for all options
+            tariffs_data = self._build_tariffs_summary(tariffs_available, for_date)
+
+            results[category_id] = [{
+                'date': for_date,
+                'gross_rate': float(gross_rate),
+                'net_rate': float(net_rate),
+                'tariffs_data': tariffs_data
+            }]
+
+        return results
+

backend/services/reconciliation_service.py

@@ -0,0 +1,531 @@
+"""
+Reconciliation Business Logic Service
+
+Ported from the WordPress plugin hotel-cashup-reconciliation.
+Handles payment categorization, variance calculation, and report aggregation.
+"""
+import re
+import logging
+from datetime import date, datetime
+from typing import List, Dict, Optional, Any
+from decimal import Decimal
+
+logger = logging.getLogger(__name__)
+
+
+# ============================================
+# PAYMENT CATEGORIZATION
+# ============================================
+
+def identify_card_type(transaction: dict) -> str:
+    """
+    Categorize a Newbook transaction into a card type.
+
+    Ported from PHP: HCR_Newbook_API::identify_card_type()
+
+    Returns: 'cash', 'visa_mc', 'amex', 'bacs', or 'other'
+    """
+    # Handle both old 'type' field and new 'payment_type' field
+    ptype = (transaction.get('payment_type') or transaction.get('type') or '').lower()
+    method = (transaction.get('method') or '').lower()
+    transaction_method = (transaction.get('payment_transaction_method') or '').lower()
+    combined = f"{ptype} {method}"
+
+    # Cash must be identified first
+    if 'cash' in combined:
+        return 'cash'
+
+    # BACS/Bank transfers
+    if any(kw in combined for kw in ['eft', 'bacs', 'bank transfer', 'banktransfer', 'direct debit']):
+        return 'bacs'
+
+    # Amex - must be explicitly identified
+    if 'amex' in combined or 'american express' in combined:
+        return 'amex'
+
+    # Visa/Mastercard - must be explicitly identified
+    if any(kw in combined for kw in ['visa', 'mastercard', 'master card', 'mc']):
+        return 'visa_mc'
+
+    # For gateway/automated transactions, default to visa_mc (most common card type)
+    if transaction_method in ('automated', 'gateway', 'cc_gateway'):
+        if any(kw in combined for kw in ['card', 'credit', 'debit']):
+            return 'visa_mc'
+        # Gateway transactions are almost always card payments
+        return 'visa_mc'
+
+    if ptype:
+        logger.warning(f"Unidentified payment type: '{ptype}' (method: '{method}', transaction_method: '{transaction_method}')")
+
+    return 'other'
+
+
+def convert_newbook_amount(amount: float) -> float:
+    """
+    Convert Newbook amount from accounting perspective to revenue perspective.
+
+    In Newbook: payments are negative, refunds are positive.
+    For reconciliation: payments should be positive, refunds negative.
+    """
+    return -float(amount)
+
+
+def process_transaction(transaction: dict) -> Optional[dict]:
+    """
+    Process a single Newbook transaction into a payment record.
+
+    Returns None if the transaction should be skipped.
+    """
+    item_type = transaction.get('item_type', '')
+
+    # Only process payments, refunds, and voided transactions
+    if item_type not in ('payments_raised', 'refunds_raised', 'payments_voided', 'refunds_voided'):
+        return None
+
+    # Skip balance transfers (system-generated, always net to zero)
+    payment_type = transaction.get('payment_type', '')
+    if payment_type == 'balance_transfer':
+        return None
+
+    amount = convert_newbook_amount(float(transaction.get('item_amount', 0)))
+
+    return {
+        'payment_id': transaction.get('item_id', ''),
+        'booking_id': str(transaction.get('booking_id', '')),
+        'guest_name': transaction.get('account_for_name', ''),
+        'payment_date': transaction.get('item_date', ''),
+        'payment_type': payment_type,
+        'payment_method': '',
+        'transaction_method': transaction.get('payment_transaction_method', 'manual'),
+        'card_type': identify_card_type(transaction),
+        'amount': amount,
+        'tendered': 0,
+        'processed_by': '',
+        'item_type': item_type,
+        'description': transaction.get('item_description', ''),
+    }
+
+
+def categorize_payments(raw_transactions: List[dict]) -> List[dict]:
+    """
+    Process raw Newbook API transactions into categorized payment records.
+
+    Filters out non-payment items and balance transfers, converts amounts,
+    and identifies card types.
+    """
+    payments = []
+    for transaction in raw_transactions:
+        payment = process_transaction(transaction)
+        if payment is not None:
+            payments.append(payment)
+    return payments
+
+
+def calculate_payment_totals(payments: List[dict]) -> dict:
+    """
+    Calculate payment totals by reconciliation category.
+
+    Ported from PHP: HCR_Newbook_API::calculate_payment_totals()
+
+    Categories:
+    - cash: Physical cash payments
+    - manual_visa_mc: Card machine (PDQ) Visa/MC payments
+    - manual_amex: Card machine (PDQ) Amex payments
+    - gateway_visa_mc: Online/gateway Visa/MC payments
+    - gateway_amex: Online/gateway Amex payments
+    - bacs: Bank transfers
+    """
+    totals = {
+        'cash': 0.0,
+        'manual_visa_mc': 0.0,
+        'manual_amex': 0.0,
+        'gateway_visa_mc': 0.0,
+        'gateway_amex': 0.0,
+        'bacs': 0.0
+    }
+
+    for payment in payments:
+        amount = float(payment.get('amount', 0))
+        transaction_method = (payment.get('transaction_method') or '').lower()
+        card_type = payment.get('card_type', '')
+
+        if card_type == 'cash':
+            totals['cash'] += amount
+        elif card_type == 'bacs':
+            totals['bacs'] += amount
+        elif transaction_method == 'manual':
+            if card_type == 'amex':
+                totals['manual_amex'] += amount
+            elif card_type == 'visa_mc':
+                totals['manual_visa_mc'] += amount
+        elif transaction_method in ('automated', 'gateway', 'cc_gateway'):
+            if card_type == 'amex':
+                totals['gateway_amex'] += amount
+            elif card_type == 'visa_mc':
+                totals['gateway_visa_mc'] += amount
+
+    # Round all totals to 2 decimal places
+    return {k: round(v, 2) for k, v in totals.items()}
+
+
+# ============================================
+# TILL SYSTEM TRANSACTIONS
+# ============================================
+
+def parse_till_transactions(raw_transactions: List[dict]) -> dict:
+    """
+    Parse till system transactions from Newbook transaction data.
+    Extracts transactions where method is "manual" and item_description follows:
+    "Ticket: {number} - {payment_type}"
+
+    Returns dict grouped by payment type with count and total.
+    """
+    till_payments = {}
+    ticket_pattern = re.compile(r'^Ticket:\s*(\d+)\s*-\s*(.+)$', re.IGNORECASE)
+
+    for transaction in raw_transactions:
+        item_type = transaction.get('item_type', '')
+        if item_type not in ('payments_raised', 'refunds_raised', 'payments_voided', 'refunds_voided'):
+            continue
+
+        method = transaction.get('payment_transaction_method', '')
+        if method != 'manual':
+            continue
+
+        description = transaction.get('item_description', '')
+        match = ticket_pattern.match(description)
+        if not match:
+            continue
+
+        payment_type = match.group(2).strip()
+
+        # Skip balance transfers
+        if payment_type == 'balance_transfer':
+            continue
+
+        amount = convert_newbook_amount(float(transaction.get('item_amount', 0)))
+        if amount == 0:
+            continue
+
+        if payment_type not in till_payments:
+            till_payments[payment_type] = {
+                'payment_type': payment_type,
+                'quantity': 0,
+                'total': 0.0,
+                'transactions': []
+            }
+
+        till_payments[payment_type]['quantity'] += 1
+        till_payments[payment_type]['total'] += amount
+        till_payments[payment_type]['transactions'].append({
+            'ticket': match.group(1),
+            'amount': amount,
+            'item_type': item_type
+        })
+
+    # Round totals
+    for key in till_payments:
+        till_payments[key]['total'] = round(till_payments[key]['total'], 2)
+
+    return till_payments
+
+
+# ============================================
+# TRANSACTION BREAKDOWN
+# ============================================
+
+def build_transaction_breakdown(payments: List[dict]) -> dict:
+    """
+    Group processed payments into a transaction breakdown for display.
+
+    Groups:
+    - reception_manual: Manual payments at reception (PDQ entered by staff)
+    - reception_gateway: Automated/gateway payments at reception
+    - restaurant_bar: Payments from till system (description contains "Ticket:")
+
+    Each group is further sub-grouped by payment type label.
+    Returns dict of groups, each containing sub-groups with transaction lists.
+    """
+    ticket_pattern = re.compile(r'Ticket:\s*(\d+)\s*-\s*(.+)', re.IGNORECASE)
+
+    reception_manual: Dict[str, list] = {}
+    reception_gateway: Dict[str, list] = {}
+    restaurant_bar: Dict[str, list] = {}
+
+    for p in payments:
+        transaction_method = (p.get('transaction_method') or '').lower()
+        card_type = p.get('card_type', 'other')
+        payment_type = p.get('payment_type', '')
+        item_type = p.get('item_type', '')
+        amount = float(p.get('amount', 0))
+        guest_name = p.get('guest_name', '')
+        payment_date = p.get('payment_date', '')
+        description = p.get('description', '')
+        is_voided = item_type in ('payments_voided', 'refunds_voided')
+
+        # Extract time from date string
+        time_str = ''
+        if payment_date and ' ' in str(payment_date):
+            time_str = str(payment_date).split(' ')[1][:5]  # HH:MM
+
+        # Determine display type label
+        type_label = payment_type.title() if payment_type else 'Other'
+        if card_type == 'cash':
+            type_label = 'Cash'
+        elif card_type == 'bacs':
+            type_label = 'BACS'
+        elif card_type == 'amex':
+            type_label = 'Amex'
+        elif card_type == 'visa_mc':
+            type_label = 'Card'
+
+        # Check for restaurant/bar till ticket pattern in description
+        ticket_match = ticket_pattern.search(description) if description else None
+        details = guest_name
+        if ticket_match:
+            ticket_num = ticket_match.group(1)
+            ticket_type = ticket_match.group(2).strip()
+            details = f"Ticket #{ticket_num} - {ticket_type}"
+            type_label = ticket_type.title() if ticket_type else type_label
+
+        entry = {
+            'time': time_str,
+            'type': type_label,
+            'details': details,
+            'amount': round(amount, 2),
+            'is_voided': is_voided,
+            'is_refund': item_type in ('refunds_raised', 'refunds_voided'),
+            'item_type': item_type,
+            'payment_id': p.get('payment_id', ''),
+            'booking_id': p.get('booking_id', ''),
+        }
+
+        # Route to appropriate group
+        if ticket_match:
+            if type_label not in restaurant_bar:
+                restaurant_bar[type_label] = []
+            restaurant_bar[type_label].append(entry)
+        elif transaction_method in ('automated', 'gateway', 'cc_gateway'):
+            if type_label not in reception_gateway:
+                reception_gateway[type_label] = []
+            reception_gateway[type_label].append(entry)
+        else:
+            # Manual and default go to reception_manual
+            if type_label not in reception_manual:
+                reception_manual[type_label] = []
+            reception_manual[type_label].append(entry)
+
+    # Calculate subtotals for each group
+    def with_subtotals(group: Dict[str, list]) -> dict:
+        result = {}
+        group_total = 0.0
+        group_count = 0
+        for key, transactions in group.items():
+            subtotal = round(sum(t['amount'] for t in transactions), 2)
+            result[key] = {
+                'transactions': transactions,
+                'subtotal': subtotal,
+                'count': len(transactions),
+            }
+            group_total += subtotal
+            group_count += len(transactions)
+        return {'groups': result, 'total': round(group_total, 2), 'count': group_count}
+
+    return {
+        'reception_manual': with_subtotals(reception_manual),
+        'reception_gateway': with_subtotals(reception_gateway),
+        'restaurant_bar': with_subtotals(restaurant_bar),
+    }
+
+
+# ============================================
+# VARIANCE CALCULATION
+# ============================================
+
+def calculate_variance(banked: float, reported: float) -> float:
+    """
+    Calculate variance between banked (manual count) and reported (Newbook).
+
+    Positive = over (extra cash/payments found)
+    Negative = short (missing cash/payments)
+    """
+    return round(banked - reported, 2)
+
+
+def get_variance_status(variance: float, threshold: float = 10.0) -> str:
+    """
+    Determine variance status for display.
+
+    Returns: 'balanced', 'over', or 'short'
+    """
+    if abs(variance) <= threshold:
+        return 'balanced'
+    elif variance > 0:
+        return 'over'
+    else:
+        return 'short'
+
+
+def build_reconciliation_rows(
+    banked_totals: dict,
+    reported_totals: dict
+) -> List[dict]:
+    """
+    Build reconciliation comparison rows for each category.
+
+    banked_totals: From manual entry (cash count + card machines)
+    reported_totals: From Newbook payments
+
+    Returns list of rows with category, banked, reported, variance.
+    """
+    categories = [
+        ('Cash', 'cash'),
+        ('PDQ Visa/MC', 'manual_visa_mc'),
+        ('PDQ Amex', 'manual_amex'),
+        ('Gateway Visa/MC', 'gateway_visa_mc'),
+        ('Gateway Amex', 'gateway_amex'),
+        ('BACS', 'bacs'),
+    ]
+
+    rows = []
+    for label, key in categories:
+        banked = banked_totals.get(key, 0.0)
+        reported = reported_totals.get(key, 0.0)
+        variance = calculate_variance(banked, reported)
+        rows.append({
+            'category': label,
+            'key': key,
+            'banked_amount': round(banked, 2),
+            'reported_amount': round(reported, 2),
+            'variance': variance,
+            'status': get_variance_status(variance)
+        })
+
+    return rows
+
+
+# ============================================
+# MULTI-DAY REPORT AGGREGATION
+# ============================================
+
+def build_multi_day_report(
+    cash_ups: List[dict],
+    payment_totals_by_date: Dict[str, dict],
+    daily_stats: List[dict],
+    sales_breakdown: List[dict],
+) -> dict:
+    """
+    Build multi-day report with 3 tables:
+    1. Daily Reconciliation Summary (banked vs reported by category per day)
+    2. Sales Breakdown (GL categories vs days)
+    3. Occupancy Stats (rooms, people, rates per day)
+
+    Returns dict with three table datasets.
+    """
+    # Table 1: Daily Reconciliation Summary
+    recon_summary = []
+    total_banked = {
+        'cash': 0, 'manual_visa_mc': 0, 'manual_amex': 0,
+        'gateway_visa_mc': 0, 'gateway_amex': 0, 'bacs': 0
+    }
+    total_reported = {
+        'cash': 0, 'manual_visa_mc': 0, 'manual_amex': 0,
+        'gateway_visa_mc': 0, 'gateway_amex': 0, 'bacs': 0
+    }
+
+    for cash_up in cash_ups:
+        date_str = cash_up['session_date']
+        reported = payment_totals_by_date.get(date_str, {})
+
+        # Build banked totals from cash_up data
+        banked = {
+            'cash': float(cash_up.get('total_cash_counted', 0)),
+            'manual_visa_mc': 0.0,
+            'manual_amex': 0.0,
+            'gateway_visa_mc': 0.0,
+            'gateway_amex': 0.0,
+            'bacs': 0.0
+        }
+
+        # Card machine totals from cash_up
+        for card in cash_up.get('card_machines', []):
+            machine_name = card.get('machine_name', '').lower()
+            banked['manual_visa_mc'] += float(card.get('visa_mc_amount', 0))
+            banked['manual_amex'] += float(card.get('amex_amount', 0))
+
+        # Reported amounts from Newbook
+        reported_amounts = {
+            'cash': float(reported.get('cash', 0)),
+            'manual_visa_mc': float(reported.get('manual_visa_mc', 0)),
+            'manual_amex': float(reported.get('manual_amex', 0)),
+            'gateway_visa_mc': float(reported.get('gateway_visa_mc', 0)),
+            'gateway_amex': float(reported.get('gateway_amex', 0)),
+            'bacs': float(reported.get('bacs', 0)),
+        }
+
+        # Calculate row variances
+        row_variance = {}
+        for key in banked:
+            row_variance[key] = round(banked[key] - reported_amounts[key], 2)
+            total_banked[key] += banked[key]
+            total_reported[key] += reported_amounts[key]
+
+        recon_summary.append({
+            'date': date_str,
+            'status': cash_up.get('status', ''),
+            'banked': {k: round(v, 2) for k, v in banked.items()},
+            'reported': {k: round(v, 2) for k, v in reported_amounts.items()},
+            'variance': row_variance,
+            'banked_total': round(sum(banked.values()), 2),
+            'reported_total': round(sum(reported_amounts.values()), 2),
+        })
+
+    # Totals row
+    total_variance = {}
+    for key in total_banked:
+        total_variance[key] = round(total_banked[key] - total_reported[key], 2)
+
+    recon_totals = {
+        'banked': {k: round(v, 2) for k, v in total_banked.items()},
+        'reported': {k: round(v, 2) for k, v in total_reported.items()},
+        'variance': total_variance,
+        'banked_total': round(sum(total_banked.values()), 2),
+        'reported_total': round(sum(total_reported.values()), 2),
+    }
+
+    # Table 2: Sales Breakdown
+    sales_by_date = {}
+    all_categories = set()
+    for row in sales_breakdown:
+        d = row['business_date']
+        cat = row['category']
+        amt = float(row['net_amount'])
+        all_categories.add(cat)
+        if d not in sales_by_date:
+            sales_by_date[d] = {}
+        sales_by_date[d][cat] = amt
+
+    # Table 3: Occupancy Stats
+    occupancy_data = []
+    for stat in daily_stats:
+        occupancy_data.append({
+            'date': stat['business_date'],
+            'gross_sales': float(stat.get('gross_sales', 0)),
+            'rooms_sold': int(stat.get('rooms_sold', 0)),
+            'total_people': int(stat.get('total_people', 0)),
+            'debtors_creditors': float(stat.get('debtors_creditors_balance', 0)),
+        })
+
+    return {
+        'reconciliation_summary': {
+            'rows': recon_summary,
+            'totals': recon_totals,
+        },
+        'sales_breakdown': {
+            'categories': sorted(list(all_categories)),
+            'by_date': sales_by_date,
+        },
+        'occupancy': {
+            'rows': occupancy_data,
+        }
+    }

backend/services/resos_client.py

@@ -0,0 +1,177 @@
+"""
+Resos API Client
+
+CRITICAL: This client is READ-ONLY. All methods use GET requests only.
+NO data is written, modified, or deleted in Resos.
+Data flows ONE WAY: Resos → Local Database
+"""
+import os
+import httpx
+import base64
+import asyncio
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ResosAPIError(Exception):
+    """Custom exception for Resos API errors"""
+    pass
+
+
+class ResosClient:
+    """
+    Async client for Resos API
+
+    Rate limiting: ~60 requests/min, using 1s delay between requests
+    Pagination: Uses skip/limit, max 100 per request
+    Date filtering: Uses fromDateTime/toDateTime
+    """
+
+    BASE_URL = "https://api.resos.com/v1"
+
+    def __init__(self, api_key: str = None):
+        # Use provided credentials or fall back to environment variables
+        self.api_key = api_key or os.getenv("RESOS_API_KEY")
+        if self.api_key:
+            # HTTP Basic Auth: base64_encode(api_key + ':')
+            self.auth_header = f"Basic {base64.b64encode(f'{self.api_key}:'.encode()).decode()}"
+        else:
+            self.auth_header = None
+            logger.warning("Resos API key not configured")
+
+    @classmethod
+    async def from_db(cls, db):
+        """Create client with credentials from database"""
+        from api.config import _get_config_value
+
+        api_key = await _get_config_value(db, "resos_api_key")
+        return cls(api_key=api_key)
+
+    async def __aenter__(self):
+        self.client = httpx.AsyncClient(timeout=30.0)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.client.aclose()
+
+    async def test_connection(self) -> bool:
+        """Test API connection by fetching opening hours"""
+        try:
+            response = await self.client.get(
+                f"{self.BASE_URL}/openingHours",
+                headers={"Authorization": self.auth_header}
+            )
+            return response.status_code == 200
+        except Exception as e:
+            logger.error(f"Resos connection test failed: {e}")
+            return False
+
+    async def get_bookings(
+        self,
+        from_date: date,
+        to_date: date
+    ) -> List[dict]:
+        """
+        Fetch bookings for date range with pagination and rate limiting.
+
+        Returns list of booking objects with structure:
+        {
+            '_id': 'booking_id',
+            'date': '2026-01-20',
+            'time': '19:00',
+            'people': 2,
+            'status': 'confirmed',
+            'source': 'website',
+            'guest': {...},
+            'customFields': [...],
+            'restaurantNotes': [...]
+        }
+        """
+        all_bookings = []
+        offset = 0
+
+        from_datetime = f"{from_date}T00:00:00"
+        to_datetime = f"{to_date}T23:59:59"
+
+        while True:
+            logger.info(f"Fetching Resos bookings: {from_date} to {to_date} (offset: {offset})")
+
+            response = await self.client.get(
+                f"{self.BASE_URL}/bookings",
+                headers={"Authorization": self.auth_header},
+                params={
+                    "fromDateTime": from_datetime,
+                    "toDateTime": to_datetime,
+                    "limit": 100,
+                    "skip": offset
+                }
+            )
+
+            if response.status_code != 200:
+                error_body = response.text
+                logger.error(f"Resos API error {response.status_code}: {error_body}")
+                raise ResosAPIError(f"Failed to fetch bookings: {response.status_code} - {error_body}")
+
+            data = response.json()
+            page_bookings = data if isinstance(data, list) else []
+
+            if not page_bookings:
+                break
+
+            all_bookings.extend(page_bookings)
+            logger.info(f"Fetched {len(page_bookings)} bookings (offset {offset})")
+
+            # If we got fewer than the limit, we've reached the end
+            if len(page_bookings) < 100:
+                break
+
+            offset += 100
+
+            # Rate limiting: 1 request per second
+            await asyncio.sleep(1)
+
+        logger.info(f"Total bookings fetched: {len(all_bookings)}")
+        return all_bookings
+
+    async def get_opening_hours(self) -> List[dict]:
+        """
+        Fetch opening hours/service periods
+
+        Returns list of opening hour objects:
+        {
+            '_id': 'opening_hour_id',
+            'name': 'Dinner',
+            'startTime': '18:00',
+            'endTime': '22:00',
+            'days': ['monday', 'tuesday', 'wednesday', ...]
+        }
+        """
+        response = await self.client.get(
+            f"{self.BASE_URL}/openingHours",
+            headers={"Authorization": self.auth_header},
+            params={"showDeleted": "false", "onlySpecial": "false"}
+        )
+
+        if response.status_code != 200:
+            raise ResosAPIError(f"Failed to fetch opening hours: {response.status_code}")
+
+        return response.json()
+
+    async def get_custom_field_definitions(self) -> List[dict]:
+        """
+        Fetch custom field definitions
+
+        Returns field definitions with choice options for dropdowns/radios
+        """
+        response = await self.client.get(
+            f"{self.BASE_URL}/customFields",
+            headers={"Authorization": self.auth_header}
+        )
+
+        if response.status_code != 200:
+            raise ResosAPIError(f"Failed to fetch custom fields: {response.status_code}")
+
+        return response.json()

backend/services/scraper_backends/__init__.py

@@ -0,0 +1,20 @@
+"""
+Scraper backends for booking.com rate scraping.
+
+Provides pluggable backends to allow switching between:
+- playwright_local: Direct Playwright (default)
+- playwright_proxy: Playwright with rotating proxies (future)
+- apify_backend: Apify scraping service (future)
+"""
+
+from .base import ScraperBackend, ScraperResult, HotelData, RateData, AvailabilityStatus
+from .playwright_local import PlaywrightLocalBackend
+
+__all__ = [
+    'ScraperBackend',
+    'ScraperResult',
+    'HotelData',
+    'RateData',
+    'AvailabilityStatus',
+    'PlaywrightLocalBackend',
+]

backend/services/scraper_backends/base.py

@@ -0,0 +1,152 @@
+"""
+Abstract base class for booking.com scraper backends.
+
+Defines the interface that all scraper backends must implement,
+allowing easy switching between local Playwright, proxied Playwright,
+or external services like Apify.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import date
+from decimal import Decimal
+from typing import List, Optional, Dict, Any
+from enum import Enum
+
+
+class AvailabilityStatus(str, Enum):
+    """Availability status for a hotel rate."""
+    AVAILABLE = 'available'      # Rate found, bookable
+    SOLD_OUT = 'sold_out'        # Hotel shows no availability
+    NO_DATA = 'no_data'          # Couldn't determine (scraper issue)
+
+
+@dataclass
+class RateData:
+    """Rate data for a single hotel on a single date."""
+    hotel_id: Optional[str] = None       # Our internal hotel_id (filled after DB lookup)
+    booking_com_id: str = ''             # Hotel ID from booking.com
+    rate_date: date = None
+    availability_status: AvailabilityStatus = AvailabilityStatus.NO_DATA
+    rate_gross: Optional[Decimal] = None
+    currency: str = 'GBP'
+    room_type: Optional[str] = None
+    breakfast_included: Optional[bool] = None
+    free_cancellation: Optional[bool] = None
+    no_prepayment: Optional[bool] = None
+    rooms_left: Optional[int] = None     # "Only X rooms left"
+    available_qty: Optional[int] = None  # Future: from hotel page dropdown
+
+
+@dataclass
+class HotelData:
+    """Hotel data discovered from search results."""
+    booking_com_id: str
+    name: str
+    booking_com_url: Optional[str] = None
+    star_rating: Optional[Decimal] = None
+    review_score: Optional[Decimal] = None
+    review_count: Optional[int] = None
+
+
+@dataclass
+class ScraperResult:
+    """Result from a scraping operation."""
+    success: bool
+    blocked: bool = False                # True if anti-scrape blocking detected
+    block_reason: Optional[str] = None   # CAPTCHA, rate limit, etc.
+    hotels: List[HotelData] = field(default_factory=list)
+    rates: List[RateData] = field(default_factory=list)
+    error_message: Optional[str] = None
+    page_content_sample: Optional[str] = None  # For debugging
+
+
+class ScraperBackend(ABC):
+    """
+    Abstract base class for scraper backends.
+
+    All backends must implement these methods to provide a consistent
+    interface for the main booking_scraper.py service.
+    """
+
+    # Common block detection signals
+    BLOCK_SIGNALS = [
+        'captcha',
+        'unusual traffic',
+        'access denied',
+        'please verify',
+        'too many requests',
+        'are you a robot',
+        'verify you are human',
+        'security check',
+    ]
+
+    @abstractmethod
+    async def scrape_location_search(
+        self,
+        location: str,
+        check_in: date,
+        check_out: date,
+        adults: int = 2,
+        pages: int = 2
+    ) -> ScraperResult:
+        """
+        Scrape booking.com location search results.
+
+        Args:
+            location: Location name (e.g., "Bowness-on-Windermere")
+            check_in: Check-in date
+            check_out: Check-out date (typically check_in + 1 for single night)
+            adults: Number of adults for search
+            pages: Number of search result pages to scrape
+
+        Returns:
+            ScraperResult with hotels and rates found
+        """
+        pass
+
+    @abstractmethod
+    async def scrape_hotel_page(
+        self,
+        hotel_url: str,
+        check_in: date,
+        check_out: date,
+        adults: int = 2
+    ) -> ScraperResult:
+        """
+        Scrape an individual hotel page for detailed rates.
+
+        Future expansion - not used in initial implementation.
+        Will provide available_qty from room dropdowns.
+
+        Args:
+            hotel_url: Full booking.com URL for the hotel
+            check_in: Check-in date
+            check_out: Check-out date
+            adults: Number of adults
+
+        Returns:
+            ScraperResult with detailed rate information
+        """
+        pass
+
+    @abstractmethod
+    async def close(self):
+        """Clean up any resources (browser instances, etc.)."""
+        pass
+
+    def detect_blocking(self, page_content: str) -> tuple[bool, Optional[str]]:
+        """
+        Check if page content shows anti-scrape response.
+
+        Args:
+            page_content: HTML content of the page
+
+        Returns:
+            Tuple of (is_blocked, reason)
+        """
+        content_lower = page_content.lower()
+        for signal in self.BLOCK_SIGNALS:
+            if signal in content_lower:
+                return True, signal
+        return False, None

backend/services/scraper_backends/playwright_local.py

@@ -0,0 +1,401 @@
+"""
+Local Playwright backend for booking.com scraping.
+
+Uses Playwright with Chromium to scrape search results.
+No proxy - direct connection. Suitable for low-volume scraping.
+"""
+
+import asyncio
+import logging
+import random
+import re
+from datetime import date
+from decimal import Decimal, InvalidOperation
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page
+
+from .base import (
+    ScraperBackend,
+    ScraperResult,
+    HotelData,
+    RateData,
+    AvailabilityStatus
+)
+
+logger = logging.getLogger(__name__)
+
+
+class PlaywrightLocalBackend(ScraperBackend):
+    """
+    Local Playwright backend using Chromium.
+
+    Features:
+    - Rotates user agents
+    - Random delays between requests
+    - Mimics human scroll behavior
+    - Uses data-testid selectors for stability
+    """
+
+    USER_AGENTS = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
+    ]
+
+    def __init__(self, proxy_config: dict = None):
+        """
+        Initialize the backend.
+
+        Args:
+            proxy_config: Optional proxy configuration (for future use)
+        """
+        self.proxy_config = proxy_config
+        self._playwright = None
+        self._browser: Optional[Browser] = None
+
+    async def _ensure_browser(self) -> Browser:
+        """Ensure browser is running, start if needed."""
+        if self._browser is None or not self._browser.is_connected():
+            self._playwright = await async_playwright().start()
+            self._browser = await self._playwright.chromium.launch(
+                headless=True,
+                args=[
+                    '--disable-blink-features=AutomationControlled',
+                    '--no-sandbox',
+                    '--disable-dev-shm-usage',
+                ]
+            )
+        return self._browser
+
+    async def _create_context(self) -> BrowserContext:
+        """Create a new browser context with random user agent."""
+        browser = await self._ensure_browser()
+        context = await browser.new_context(
+            user_agent=random.choice(self.USER_AGENTS),
+            viewport={'width': 1920, 'height': 1080},
+            locale='en-GB',
+            timezone_id='Europe/London',
+        )
+        return context
+
+    def _build_search_url(
+        self,
+        location: str,
+        check_in: date,
+        check_out: date,
+        adults: int,
+        offset: int = 0
+    ) -> str:
+        """Build booking.com search URL with parameters."""
+        params = {
+            'ss': location,
+            'checkin': check_in.isoformat(),
+            'checkout': check_out.isoformat(),
+            'group_adults': adults,
+            'no_rooms': 1,
+            'group_children': 0,
+        }
+        if offset > 0:
+            params['offset'] = offset
+
+        return f"https://www.booking.com/searchresults.en-gb.html?{urlencode(params)}"
+
+    def _parse_price(self, price_text: str) -> Optional[Decimal]:
+        """Parse price from text like '£150' or 'GBP 150'."""
+        if not price_text:
+            return None
+        # Remove currency symbols and extract number
+        cleaned = re.sub(r'[£$€,\s]', '', price_text)
+        # Find first number (including decimals)
+        match = re.search(r'[\d,]+(?:\.\d{2})?', cleaned)
+        if match:
+            try:
+                return Decimal(match.group().replace(',', ''))
+            except InvalidOperation:
+                return None
+        return None
+
+    def _extract_hotel_id(self, url: str) -> Optional[str]:
+        """Extract hotel ID from booking.com URL."""
+        if not url:
+            return None
+        # URL format: /hotel/gb/hotel-name.en-gb.html or ?dest_id=123
+        # Try to extract from URL path
+        match = re.search(r'/hotel/[a-z]{2}/([^/]+)\.', url)
+        if match:
+            return match.group(1)
+        # Try dest_id parameter
+        match = re.search(r'dest_id=(-?\d+)', url)
+        if match:
+            return match.group(1)
+        return None
+
+    async def _human_like_scroll(self, page: Page):
+        """Simulate human-like scrolling behavior."""
+        # Scroll down in increments
+        for _ in range(3):
+            await page.mouse.wheel(0, random.randint(300, 600))
+            await asyncio.sleep(random.uniform(0.3, 0.8))
+
+    async def _extract_search_results(self, page: Page, rate_date: date) -> tuple[List[HotelData], List[RateData]]:
+        """Extract hotel and rate data from search results page."""
+        hotels = []
+        rates = []
+
+        # Wait for property cards - booking.com uses data-testid
+        try:
+            await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
+        except Exception as e:
+            logger.warning(f"No property cards found: {e}")
+            return hotels, rates
+
+        # Get all property cards
+        cards = await page.query_selector_all('[data-testid="property-card"]')
+        logger.info(f"Found {len(cards)} property cards")
+
+        for card in cards:
+            try:
+                hotel = HotelData(booking_com_id='', name='')
+                rate = RateData(rate_date=rate_date)
+
+                # Hotel name
+                name_el = await card.query_selector('[data-testid="title"]')
+                if name_el:
+                    hotel.name = (await name_el.inner_text()).strip()
+
+                if not hotel.name:
+                    continue  # Skip if no name found
+
+                # Hotel URL and ID
+                link_el = await card.query_selector('[data-testid="title-link"]')
+                if link_el:
+                    hotel.booking_com_url = await link_el.get_attribute('href')
+                    hotel.booking_com_id = self._extract_hotel_id(hotel.booking_com_url) or ''
+
+                rate.booking_com_id = hotel.booking_com_id
+
+                # Star rating - look for star icons or rating text
+                stars_el = await card.query_selector('[data-testid="rating-stars"]')
+                if stars_el:
+                    stars_text = await stars_el.get_attribute('aria-label') or ''
+                    match = re.search(r'(\d+)', stars_text)
+                    if match:
+                        hotel.star_rating = Decimal(match.group(1))
+
+                # Review score
+                score_el = await card.query_selector('[data-testid="review-score"]')
+                if score_el:
+                    score_text = await score_el.inner_text()
+                    match = re.search(r'([\d.]+)', score_text)
+                    if match:
+                        try:
+                            hotel.review_score = Decimal(match.group(1))
+                        except InvalidOperation:
+                            pass
+
+                # Check for no availability message FIRST
+                no_avail_el = await card.query_selector('[data-testid="availability-message"]')
+                if no_avail_el:
+                    avail_text = (await no_avail_el.inner_text()).lower()
+                    if 'no availability' in avail_text or 'sold out' in avail_text:
+                        rate.availability_status = AvailabilityStatus.SOLD_OUT
+                        hotels.append(hotel)
+                        rates.append(rate)
+                        continue
+
+                # Price
+                price_el = await card.query_selector('[data-testid="price-and-discounted-price"]')
+                if not price_el:
+                    # Try alternative selector
+                    price_el = await card.query_selector('[data-testid="price"]')
+
+                if price_el:
+                    price_text = await price_el.inner_text()
+                    rate.rate_gross = self._parse_price(price_text)
+                    if rate.rate_gross:
+                        rate.availability_status = AvailabilityStatus.AVAILABLE
+
+                # Room type
+                room_el = await card.query_selector('[data-testid="recommended-units"]')
+                if room_el:
+                    rate.room_type = (await room_el.inner_text()).strip()
+
+                # Rate option badges - try multiple selectors
+                # Breakfast included
+                breakfast_el = await card.query_selector('[data-testid="breakfast-included"]')
+                if not breakfast_el:
+                    # Check text content for breakfast mentions
+                    card_text = (await card.inner_text()).lower()
+                    rate.breakfast_included = 'breakfast included' in card_text
+                else:
+                    rate.breakfast_included = True
+
+                # Free cancellation
+                cancel_el = await card.query_selector('[data-testid="cancellation-policy"]')
+                if cancel_el:
+                    cancel_text = (await cancel_el.inner_text()).lower()
+                    rate.free_cancellation = 'free cancellation' in cancel_text
+                else:
+                    card_text = (await card.inner_text()).lower()
+                    rate.free_cancellation = 'free cancellation' in card_text
+
+                # No prepayment
+                prepay_el = await card.query_selector('[data-testid="no-prepayment"]')
+                if prepay_el:
+                    rate.no_prepayment = True
+                else:
+                    card_text = (await card.inner_text()).lower()
+                    rate.no_prepayment = 'no prepayment' in card_text
+
+                # Rooms left / scarcity indicator
+                scarcity_el = await card.query_selector('[data-testid="availability-rate"]')
+                if scarcity_el:
+                    scarcity_text = await scarcity_el.inner_text()
+                    match = re.search(r'(\d+)\s*room', scarcity_text.lower())
+                    if match:
+                        rate.rooms_left = int(match.group(1))
+
+                hotels.append(hotel)
+                rates.append(rate)
+
+            except Exception as e:
+                logger.warning(f"Error extracting hotel data: {e}")
+                continue
+
+        return hotels, rates
+
+    async def scrape_location_search(
+        self,
+        location: str,
+        check_in: date,
+        check_out: date,
+        adults: int = 2,
+        pages: int = 2
+    ) -> ScraperResult:
+        """
+        Scrape booking.com location search results.
+
+        Args:
+            location: Location name
+            check_in: Check-in date
+            check_out: Check-out date (check_in + 1 for single night rate)
+            adults: Number of adults
+            pages: Number of result pages to scrape
+
+        Returns:
+            ScraperResult with hotels and rates found
+        """
+        all_hotels = []
+        all_rates = []
+        seen_hotel_ids = set()
+
+        context = None
+        page = None
+
+        try:
+            context = await self._create_context()
+            page = await context.new_page()
+
+            for page_num in range(pages):
+                # Random delay between pages (3-7 seconds)
+                if page_num > 0:
+                    delay = random.uniform(3, 7)
+                    logger.info(f"Waiting {delay:.1f}s before page {page_num + 1}")
+                    await asyncio.sleep(delay)
+
+                # Build URL with offset for pagination (25 results per page)
+                url = self._build_search_url(
+                    location, check_in, check_out, adults,
+                    offset=page_num * 25
+                )
+
+                logger.info(f"Scraping page {page_num + 1}: {url}")
+
+                try:
+                    await page.goto(url, wait_until='networkidle', timeout=30000)
+                except Exception as e:
+                    logger.warning(f"Page load timeout, continuing: {e}")
+
+                # Check for blocking
+                content = await page.content()
+                is_blocked, reason = self.detect_blocking(content)
+                if is_blocked:
+                    logger.warning(f"Blocking detected: {reason}")
+                    return ScraperResult(
+                        success=False,
+                        blocked=True,
+                        block_reason=reason,
+                        hotels=all_hotels,
+                        rates=all_rates,
+                        page_content_sample=content[:1000]
+                    )
+
+                # Human-like scrolling
+                await self._human_like_scroll(page)
+
+                # Extract data
+                hotels, rates = await self._extract_search_results(page, check_in)
+
+                # Deduplicate by booking_com_id
+                for hotel, rate in zip(hotels, rates):
+                    if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids:
+                        seen_hotel_ids.add(hotel.booking_com_id)
+                        all_hotels.append(hotel)
+                        all_rates.append(rate)
+
+                logger.info(f"Page {page_num + 1}: found {len(hotels)} hotels, {len(all_hotels)} total unique")
+
+            return ScraperResult(
+                success=True,
+                blocked=False,
+                hotels=all_hotels,
+                rates=all_rates
+            )
+
+        except Exception as e:
+            logger.error(f"Scrape error: {e}")
+            return ScraperResult(
+                success=False,
+                blocked=False,
+                error_message=str(e),
+                hotels=all_hotels,
+                rates=all_rates
+            )
+        finally:
+            if page:
+                await page.close()
+            if context:
+                await context.close()
+
+    async def scrape_hotel_page(
+        self,
+        hotel_url: str,
+        check_in: date,
+        check_out: date,
+        adults: int = 2
+    ) -> ScraperResult:
+        """
+        Scrape individual hotel page for detailed rates.
+
+        Future expansion - placeholder for now.
+        Will extract available_qty from room dropdowns.
+        """
+        # Not implemented in Phase 2a
+        logger.warning("scrape_hotel_page not yet implemented")
+        return ScraperResult(
+            success=False,
+            error_message="Hotel page scraping not yet implemented"
+        )
+
+    async def close(self):
+        """Clean up browser resources."""
+        if self._browser:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright:
+            await self._playwright.stop()
+            self._playwright = None

backend/utils/__init__.py

@@ -0,0 +1,22 @@
+"""
+Backend utilities module.
+"""
+from .time_alignment import (
+    get_prior_year_daily,
+    get_prior_year_weekly,
+    get_prior_year_week_dates,
+    get_prior_year_monthly,
+    get_comparison_info,
+    SQL_PRIOR_YEAR_DAILY,
+    SQL_PRIOR_YEAR_OFFSET,
+)
+
+__all__ = [
+    'get_prior_year_daily',
+    'get_prior_year_weekly',
+    'get_prior_year_week_dates',
+    'get_prior_year_monthly',
+    'get_comparison_info',
+    'SQL_PRIOR_YEAR_DAILY',
+    'SQL_PRIOR_YEAR_OFFSET',
+]

backend/utils/capacity.py

@@ -0,0 +1,116 @@
+"""
+Room capacity utilities
+
+Functions for getting bookable room counts accounting for maintenance.
+"""
+from datetime import date
+from sqlalchemy import text
+
+
+def get_bookable_cap_sync(db, forecast_date: date = None, fallback_value: int = 25) -> int:
+    """
+    Get the bookable rooms cap for a specific date (synchronous version).
+
+    Bookable = Total Rooms - Maintenance - Allotted
+
+    Args:
+        db: Synchronous database session
+        forecast_date: Specific date to get cap for (optional)
+        fallback_value: Default if no data found
+
+    Returns:
+        Bookable room count (cap for room forecasts)
+    """
+    # First try to get specific date's bookable count from stats
+    if forecast_date:
+        result = db.execute(text("""
+            SELECT bookable_count
+            FROM newbook_bookings_stats
+            WHERE date = :target_date AND bookable_count IS NOT NULL
+        """), {"target_date": forecast_date})
+        row = result.fetchone()
+        if row and row.bookable_count is not None:
+            return int(row.bookable_count)
+
+        # Try occupancy report data for future dates
+        result = db.execute(text("""
+            SELECT
+                SUM(COALESCE(o.available, 0) - COALESCE(o.maintenance, 0)) as bookable
+            FROM newbook_occupancy_report_data o
+            JOIN newbook_room_categories c ON o.category_id = c.site_id
+            WHERE o.date = :target_date AND c.is_included = true
+        """), {"target_date": forecast_date})
+        row = result.fetchone()
+        if row and row.bookable is not None:
+            return int(row.bookable)
+
+    # Fall back to most recent bookable_count
+    result = db.execute(text("""
+        SELECT bookable_count
+        FROM newbook_bookings_stats
+        WHERE bookable_count IS NOT NULL
+        ORDER BY date DESC
+        LIMIT 1
+    """))
+    row = result.fetchone()
+    if row and row.bookable_count:
+        return int(row.bookable_count)
+
+    return fallback_value
+
+
+async def get_bookable_cap(db, forecast_date: date = None, fallback_value: int = 25) -> int:
+    """
+    Get the bookable rooms cap for a specific date.
+
+    Bookable = Total Rooms - Maintenance - Allotted
+
+    For future dates without stats data, tries occupancy report data first,
+    then falls back to most recent bookable_count from stats.
+
+    Args:
+        db: Database session
+        forecast_date: Specific date to get cap for (optional)
+        fallback_value: Default if no data found
+
+    Returns:
+        Bookable room count (cap for room forecasts)
+    """
+    # First try to get specific date's bookable count from stats
+    if forecast_date:
+        result = await db.execute(text("""
+            SELECT bookable_count
+            FROM newbook_bookings_stats
+            WHERE date = :target_date AND bookable_count IS NOT NULL
+        """), {"target_date": forecast_date})
+        row = result.fetchone()
+        # Accept 0 as valid (all rooms in maintenance)
+        if row and row.bookable_count is not None:
+            return int(row.bookable_count)
+
+        # Try occupancy report data for future dates
+        result = await db.execute(text("""
+            SELECT
+                SUM(COALESCE(o.available, 0) - COALESCE(o.maintenance, 0)) as bookable
+            FROM newbook_occupancy_report_data o
+            JOIN newbook_room_categories c ON o.category_id = c.site_id
+            WHERE o.date = :target_date AND c.is_included = true
+        """), {"target_date": forecast_date})
+        row = result.fetchone()
+        # Accept 0 as valid bookable count (all rooms in maintenance)
+        if row and row.bookable is not None:
+            return int(row.bookable)
+
+    # Fall back to most recent bookable_count
+    result = await db.execute(text("""
+        SELECT bookable_count
+        FROM newbook_bookings_stats
+        WHERE bookable_count IS NOT NULL
+        ORDER BY date DESC
+        LIMIT 1
+    """))
+    row = result.fetchone()
+    if row and row.bookable_count:
+        return int(row.bookable_count)
+
+    return fallback_value

backend/utils/time_alignment.py

@@ -0,0 +1,161 @@
+"""
+Time alignment utilities for prior year comparisons.
+
+Ensures consistent comparison logic across the application:
+- Daily: 364 days (52 weeks) for day-of-week alignment (Mon→Mon, Sat→Sat)
+- Weekly: ISO week number matching (Week 6 2026 vs Week 6 2025)
+- Monthly: Same month, prior year
+
+This module should be used whenever comparing to prior year data.
+"""
+from datetime import date, timedelta
+from typing import Tuple, Optional
+
+
+def get_prior_year_daily(target_date: date) -> date:
+    """
+    Get the comparable date from prior year for daily comparisons.
+
+    Uses 364 days (exactly 52 weeks) to ensure day-of-week alignment:
+    - Monday → Monday
+    - Saturday → Saturday
+
+    Example:
+        Wed 11 Feb 2026 → Wed 12 Feb 2025 (not 11 Feb 2025 which was a different day)
+
+    Args:
+        target_date: The date to find comparison for
+
+    Returns:
+        The prior year date with same day of week
+    """
+    return target_date - timedelta(days=364)
+
+
+def get_prior_year_weekly(target_date: date) -> Tuple[int, int]:
+    """
+    Get the ISO week and year for weekly year-over-year comparison.
+
+    Uses ISO week numbers so Week 6 of 2026 compares to Week 6 of 2025.
+    This ensures full weeks are compared (Mon-Sun) regardless of calendar dates.
+
+    Args:
+        target_date: Any date within the target week
+
+    Returns:
+        Tuple of (year, week_number) for the comparison week
+    """
+    iso_cal = target_date.isocalendar()
+    return (iso_cal.year - 1, iso_cal.week)
+
+
+def get_prior_year_week_dates(target_date: date) -> Tuple[date, date]:
+    """
+    Get the start and end dates of the same ISO week in the prior year.
+
+    Useful for querying data for the entire comparison week.
+
+    Args:
+        target_date: Any date within the target week
+
+    Returns:
+        Tuple of (week_start, week_end) for the prior year's matching week
+    """
+    iso_cal = target_date.isocalendar()
+    prior_year = iso_cal.year - 1
+    prior_week = iso_cal.week
+
+    # Handle edge case: if prior year doesn't have this week number
+    # (can happen with week 53), fall back to last week of prior year
+    try:
+        # Find the first day of the target week in prior year
+        # Week 1 day 1 of the prior year
+        jan_1_prior = date(prior_year, 1, 1)
+        jan_1_iso = jan_1_prior.isocalendar()
+
+        # Calculate days to add to get to the target week
+        # First, get to week 1 day 1
+        days_to_week_1 = (1 - jan_1_iso.weekday) % 7
+        week_1_monday = jan_1_prior + timedelta(days=days_to_week_1)
+
+        # Adjust if Jan 1 is in the previous year's last week
+        if jan_1_iso.week != 1:
+            week_1_monday = jan_1_prior + timedelta(days=(7 - jan_1_prior.weekday()))
+
+        # Now add weeks to get to target week
+        week_start = week_1_monday + timedelta(weeks=prior_week - 1)
+
+        # Verify we got the right week
+        if week_start.isocalendar().week != prior_week:
+            # Fallback: use last week of prior year
+            dec_28_prior = date(prior_year, 12, 28)  # Always in last week
+            dec_28_iso = dec_28_prior.isocalendar()
+            week_start = dec_28_prior - timedelta(days=dec_28_prior.weekday())
+
+        week_end = week_start + timedelta(days=6)
+        return (week_start, week_end)
+
+    except (ValueError, AttributeError):
+        # Fallback to simpler calculation
+        prior_date = get_prior_year_daily(target_date)
+        week_start = prior_date - timedelta(days=prior_date.weekday())
+        week_end = week_start + timedelta(days=6)
+        return (week_start, week_end)
+
+
+def get_prior_year_monthly(target_date: date) -> Tuple[date, date]:
+    """
+    Get the start and end dates of the same month in the prior year.
+
+    Args:
+        target_date: Any date within the target month
+
+    Returns:
+        Tuple of (month_start, month_end) for the prior year's matching month
+    """
+    prior_year = target_date.year - 1
+    month = target_date.month
+
+    # First day of the month
+    month_start = date(prior_year, month, 1)
+
+    # Last day of the month
+    if month == 12:
+        month_end = date(prior_year + 1, 1, 1) - timedelta(days=1)
+    else:
+        month_end = date(prior_year, month + 1, 1) - timedelta(days=1)
+
+    return (month_start, month_end)
+
+
+# SQL helper constants for use in queries
+SQL_PRIOR_YEAR_DAILY = "INTERVAL '364 days'"  # For daily comparisons
+SQL_PRIOR_YEAR_OFFSET = 364  # Days offset for daily DOW alignment
+
+
+def get_comparison_info(target_date: date) -> dict:
+    """
+    Get formatted comparison information for display.
+
+    Useful for showing users what date/week is being compared.
+
+    Args:
+        target_date: The date being forecasted/analyzed
+
+    Returns:
+        Dict with comparison details
+    """
+    prior_daily = get_prior_year_daily(target_date)
+    prior_week_year, prior_week_num = get_prior_year_weekly(target_date)
+
+    day_names = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
+
+    return {
+        "target_date": target_date,
+        "target_day": day_names[target_date.weekday()],
+        "target_iso_week": target_date.isocalendar().week,
+        "prior_year_date": prior_daily,
+        "prior_year_day": day_names[prior_daily.weekday()],  # Should match target_day
+        "prior_year_iso_week": prior_week_num,
+        "comparison_note": f"vs {day_names[prior_daily.weekday()]} {prior_daily.strftime('%d %b %Y')}"
+    }

db/init_clean.sql

@@ -0,0 +1,872 @@
+-- Forecasting Application Database Schema
+-- PostgreSQL 15+
+-- Database: forecast_data (created via POSTGRES_DB env var)
+
+-- Enable UUID extension
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- ============================================
+-- USERS & AUTHENTICATION
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS users (
+    id SERIAL PRIMARY KEY,
+    username VARCHAR(50) UNIQUE NOT NULL,
+    password_hash VARCHAR(255) NOT NULL,
+    display_name VARCHAR(100),
+    is_active BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+-- Default admin user (password: admin123 - change in production!)
+INSERT INTO users (username, password_hash, display_name) VALUES
+('admin', '$2b$12$LQv3c1yqBWVHxkd0LHAkCOYz6TtxMQJqhN8/X4.VTtYH0lHXG0Kku', 'Administrator')
+ON CONFLICT (username) DO NOTHING;
+
+-- ============================================
+-- API KEYS (for external integrations)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS api_keys (
+    id SERIAL PRIMARY KEY,
+    key_hash VARCHAR(64) NOT NULL UNIQUE,     -- SHA256 hash of key (never store plaintext)
+    key_prefix VARCHAR(20) NOT NULL,          -- First chars for display (e.g., "fk_abc123...")
+    name VARCHAR(100) NOT NULL,               -- Descriptive name (e.g., "Kitchen Flash App")
+    is_active BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMP DEFAULT NOW(),
+    last_used_at TIMESTAMP,
+    created_by VARCHAR(100)
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_keys_hash ON api_keys(key_hash);
+CREATE INDEX IF NOT EXISTS idx_api_keys_active ON api_keys(is_active);
+
+-- ============================================
+-- SYSTEM CONFIGURATION
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS system_config (
+    id SERIAL PRIMARY KEY,
+    config_key VARCHAR(100) NOT NULL UNIQUE,
+    config_value TEXT,
+    is_encrypted BOOLEAN DEFAULT FALSE,
+    description TEXT,
+    updated_at TIMESTAMP DEFAULT NOW(),
+    updated_by VARCHAR(100)
+);
+
+-- Default config entries
+INSERT INTO system_config (config_key, description) VALUES
+('newbook_api_key', 'Newbook API Key'),
+('newbook_username', 'Newbook Username'),
+('newbook_password', 'Newbook Password'),
+('newbook_region', 'Newbook Region Code'),
+('resos_api_key', 'Resos API Key'),
+('total_rooms', 'Total number of hotel rooms'),
+('hotel_name', 'Hotel/Property Name'),
+('timezone', 'Local timezone (e.g., Europe/London)'),
+('accommodation_vat_rate', 'VAT rate for accommodation (e.g., 0.20 for 20%)'),
+('sync_newbook_enabled', 'Enable automatic Newbook sync (true/false)'),
+('sync_resos_enabled', 'Enable automatic Resos sync (true/false)'),
+('sync_schedule_time', 'Time for daily sync (HH:MM format)'),
+('sync_newbook_bookings_enabled', 'Enable automatic Newbook bookings data sync (true/false)'),
+('sync_newbook_bookings_type', 'Newbook bookings sync type (incremental/full)'),
+('sync_newbook_bookings_time', 'Newbook bookings sync time (HH:MM)'),
+('sync_newbook_occupancy_enabled', 'Enable automatic Newbook occupancy report sync (true/false)'),
+('sync_newbook_occupancy_time', 'Newbook occupancy report sync time (HH:MM)'),
+('last_bookings_aggregation_at', 'Timestamp of last bookings stats aggregation'),
+('sync_newbook_earned_revenue_enabled', 'Enable automatic Newbook earned revenue sync (true/false)'),
+('sync_newbook_earned_revenue_time', 'Newbook earned revenue sync time (HH:MM)'),
+('last_revenue_aggregation_at', 'Timestamp of last revenue aggregation'),
+('sync_newbook_current_rates_enabled', 'Enable automatic Newbook current rates sync for pickup-v2 (true/false)'),
+('sync_newbook_current_rates_time', 'Newbook current rates sync time (HH:MM)')
+ON CONFLICT (config_key) DO NOTHING;
+
+-- Set defaults
+UPDATE system_config SET config_value = '80' WHERE config_key = 'total_rooms' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'Europe/London' WHERE config_key = 'timezone' AND config_value IS NULL;
+UPDATE system_config SET config_value = '0.20' WHERE config_key = 'accommodation_vat_rate' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_resos_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:00' WHERE config_key = 'sync_schedule_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_bookings_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'incremental' WHERE config_key = 'sync_newbook_bookings_type' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:00' WHERE config_key = 'sync_newbook_bookings_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_occupancy_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:00' WHERE config_key = 'sync_newbook_occupancy_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_earned_revenue_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:10' WHERE config_key = 'sync_newbook_earned_revenue_time' AND config_value IS NULL;
+UPDATE system_config SET config_value = 'false' WHERE config_key = 'sync_newbook_current_rates_enabled' AND config_value IS NULL;
+UPDATE system_config SET config_value = '05:20' WHERE config_key = 'sync_newbook_current_rates_time' AND config_value IS NULL;
+
+-- ============================================
+-- TAX RATES (date-based tax configuration)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS tax_rates (
+    id SERIAL PRIMARY KEY,
+    tax_type VARCHAR(50) NOT NULL,          -- 'accommodation_vat', 'food_vat', etc.
+    rate DECIMAL(5,4) NOT NULL,             -- e.g., 0.20 for 20%
+    effective_from DATE NOT NULL,           -- Date this rate becomes effective
+    created_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(tax_type, effective_from)
+);
+
+CREATE INDEX IF NOT EXISTS idx_tax_rates_type ON tax_rates(tax_type);
+CREATE INDEX IF NOT EXISTS idx_tax_rates_effective ON tax_rates(tax_type, effective_from);
+
+-- Default accommodation VAT rate (20% from 2022-01-01)
+INSERT INTO tax_rates (tax_type, rate, effective_from) VALUES
+('accommodation_vat', 0.20, '2022-01-01')
+ON CONFLICT (tax_type, effective_from) DO NOTHING;
+
+-- ============================================
+-- NEWBOOK ROOM CATEGORIES (for occupancy settings)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_room_categories (
+    id SERIAL PRIMARY KEY,
+    site_id VARCHAR(50) NOT NULL UNIQUE,
+    site_name VARCHAR(255) NOT NULL,
+    site_type VARCHAR(100),
+    room_count INTEGER DEFAULT 0,
+    is_included BOOLEAN DEFAULT TRUE,
+    display_order INTEGER DEFAULT 0,
+    fetched_at TIMESTAMP DEFAULT NOW()
+);
+
+-- ============================================
+-- NEWBOOK GL ACCOUNTS (for revenue mapping)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_gl_accounts (
+    id SERIAL PRIMARY KEY,
+    gl_account_id VARCHAR(50) NOT NULL UNIQUE,
+    gl_code VARCHAR(50),
+    gl_name VARCHAR(255),
+    gl_group_id VARCHAR(50),
+    gl_group_name VARCHAR(255),
+    department VARCHAR(20),  -- 'accommodation', 'dry', 'wet', or null
+    last_seen_date DATE,
+    total_amount DECIMAL(14,2) DEFAULT 0,
+    is_active BOOLEAN DEFAULT TRUE,
+    fetched_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_gl_accounts_department ON newbook_gl_accounts(department);
+CREATE INDEX IF NOT EXISTS idx_gl_accounts_group ON newbook_gl_accounts(gl_group_name);
+
+-- ============================================
+-- NEWBOOK BOOKINGS DATA (historical booking data)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_bookings_data (
+    id SERIAL PRIMARY KEY,
+    newbook_id VARCHAR(50) NOT NULL UNIQUE,
+    booking_reference VARCHAR(100),
+    bookings_group_id VARCHAR(50),
+    booking_placed TIMESTAMP,  -- When booking was created (for lead time calculations)
+    arrival_date DATE NOT NULL,
+    departure_date DATE NOT NULL,
+    nights INTEGER,
+    adults INTEGER DEFAULT 0,
+    children INTEGER DEFAULT 0,
+    infants INTEGER DEFAULT 0,
+    total_guests INTEGER,
+    category_id VARCHAR(50),
+    room_type VARCHAR(100),
+    site_id VARCHAR(50),
+    room_number VARCHAR(50),
+    status VARCHAR(50),
+    total_amount DECIMAL(12,2),
+    tariff_name VARCHAR(255),
+    tariff_total DECIMAL(12,2),
+    travel_agent_id VARCHAR(50),
+    travel_agent_name VARCHAR(255),
+    travel_agent_commission DECIMAL(12,2),
+    booking_source_id VARCHAR(50),
+    booking_source_name VARCHAR(255),
+    booking_parent_source_id VARCHAR(50),
+    booking_parent_source_name VARCHAR(255),
+    booking_method_id VARCHAR(50),
+    booking_method_name VARCHAR(100),
+    raw_json JSONB,
+    fetched_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_bookings_arrival ON newbook_bookings_data(arrival_date);
+CREATE INDEX IF NOT EXISTS idx_bookings_status ON newbook_bookings_data(status);
+CREATE INDEX IF NOT EXISTS idx_bookings_placed ON newbook_bookings_data(booking_placed);
+
+-- ============================================
+-- NEWBOOK EARNED REVENUE DATA (historical revenue data)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_earned_revenue_data (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    gl_account_id VARCHAR(50),
+    gl_code VARCHAR(50),
+    gl_name VARCHAR(255),
+    amount_gross DECIMAL(12,2) DEFAULT 0,
+    amount_net DECIMAL(12,2) DEFAULT 0,
+    revenue_type VARCHAR(30),
+    fetched_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, gl_account_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_earned_revenue_data_date ON newbook_earned_revenue_data(date);
+CREATE INDEX IF NOT EXISTS idx_earned_revenue_data_type ON newbook_earned_revenue_data(date, revenue_type);
+
+-- ============================================
+-- NEWBOOK NET REVENUE DATA (aggregated by department)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_net_revenue_data (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL UNIQUE,
+    accommodation DECIMAL(12,2) DEFAULT 0,  -- Net accommodation revenue
+    dry DECIMAL(12,2) DEFAULT 0,            -- Net dry (food) revenue
+    wet DECIMAL(12,2) DEFAULT 0,            -- Net wet (beverage) revenue
+    aggregated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_net_revenue_data_date ON newbook_net_revenue_data(date);
+
+-- ============================================
+-- NEWBOOK OCCUPANCY REPORT DATA (official capacity & occupancy)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_occupancy_report_data (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    category_id VARCHAR(50) NOT NULL,
+    category_name VARCHAR(255),
+    available INTEGER DEFAULT 0,       -- Total configured rooms for category
+    occupied INTEGER DEFAULT 0,        -- Official occupied per Newbook
+    maintenance INTEGER DEFAULT 0,     -- Rooms offline (deduct from available for bookable rooms)
+    allotted INTEGER DEFAULT 0,        -- Block allocations
+    revenue_gross DECIMAL(12,2) DEFAULT 0,
+    revenue_net DECIMAL(12,2) DEFAULT 0,
+    occupancy_pct DECIMAL(5,2),
+    fetched_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, category_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_occupancy_report_data_date ON newbook_occupancy_report_data(date);
+
+-- ============================================
+-- SYNC LOGGING
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS sync_log (
+    id SERIAL PRIMARY KEY,
+    sync_type VARCHAR(30) NOT NULL,
+    source VARCHAR(20) NOT NULL,
+    started_at TIMESTAMP NOT NULL,
+    completed_at TIMESTAMP,
+    status VARCHAR(20) NOT NULL,
+    records_fetched INTEGER,
+    records_created INTEGER,
+    records_updated INTEGER,
+    date_from DATE,
+    date_to DATE,
+    error_message TEXT,
+    triggered_by VARCHAR(100)
+);
+
+-- ============================================
+-- NEWBOOK BOOKINGS STATS (aggregated daily stats)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_bookings_stats (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL UNIQUE,
+
+    -- Room availability (from newbook_occupancy_report_data)
+    rooms_count INTEGER DEFAULT 0,           -- Total available rooms (included categories)
+    maintenance_count INTEGER DEFAULT 0,     -- Rooms offline/maintenance
+    bookable_count INTEGER DEFAULT 0,        -- rooms_count - maintenance_count
+
+    -- Occupancy totals (from bookings)
+    booking_count INTEGER DEFAULT 0,         -- Occupied rooms (bookings staying this night)
+    guests_count INTEGER DEFAULT 0,
+    adults_count INTEGER DEFAULT 0,
+    children_count INTEGER DEFAULT 0,
+    infants_count INTEGER DEFAULT 0,
+
+    -- Occupancy percentages
+    total_occupancy_pct DECIMAL(5,2),        -- booking_count / rooms_count * 100
+    bookable_occupancy_pct DECIMAL(5,2),     -- booking_count / bookable_count * 100
+
+    -- Revenue totals
+    guest_rate_total DECIMAL(12,2) DEFAULT 0,      -- SUM of calculated_amount (gross)
+    net_booking_rev_total DECIMAL(12,2) DEFAULT 0, -- SUM of net accommodation
+
+    -- Per-category breakdowns (JSONB)
+    occupancy_by_category JSONB DEFAULT '{}',
+    revenue_by_category JSONB DEFAULT '{}',
+    availability_by_category JSONB DEFAULT '{}',
+
+    -- Pickup-V2: Rate statistics per category for bounds calculation
+    -- Structure: { "category_id": { "min_net": 120, "max_net": 200, "adr_net": 155, "rooms": 12 } }
+    rate_stats_by_category JSONB DEFAULT '{}',
+
+    aggregated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_bookings_stats_date ON newbook_bookings_stats(date);
+
+-- ============================================
+-- NEWBOOK BOOKING PACE (lead-time snapshots for forecasting)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_booking_pace (
+    id SERIAL PRIMARY KEY,
+    arrival_date DATE NOT NULL UNIQUE,
+
+    -- Monthly intervals: months 7-12 (6 columns)
+    d365 INTEGER,    -- 12 months out
+    d330 INTEGER,    -- 11 months out
+    d300 INTEGER,    -- 10 months out
+    d270 INTEGER,    -- 9 months out
+    d240 INTEGER,    -- 8 months out
+    d210 INTEGER,    -- 7 months out
+
+    -- Weekly intervals: weeks 5-25 (21 columns)
+    d177 INTEGER, d170 INTEGER, d163 INTEGER, d156 INTEGER, d149 INTEGER,
+    d142 INTEGER, d135 INTEGER, d128 INTEGER, d121 INTEGER, d114 INTEGER,
+    d107 INTEGER, d100 INTEGER, d93 INTEGER, d86 INTEGER, d79 INTEGER,
+    d72 INTEGER, d65 INTEGER, d58 INTEGER, d51 INTEGER, d44 INTEGER, d37 INTEGER,
+
+    -- Daily intervals: days 0-30 (31 columns)
+    d30 INTEGER, d29 INTEGER, d28 INTEGER, d27 INTEGER, d26 INTEGER,
+    d25 INTEGER, d24 INTEGER, d23 INTEGER, d22 INTEGER, d21 INTEGER,
+    d20 INTEGER, d19 INTEGER, d18 INTEGER, d17 INTEGER, d16 INTEGER,
+    d15 INTEGER, d14 INTEGER, d13 INTEGER, d12 INTEGER, d11 INTEGER,
+    d10 INTEGER, d9 INTEGER, d8 INTEGER, d7 INTEGER, d6 INTEGER,
+    d5 INTEGER, d4 INTEGER, d3 INTEGER, d2 INTEGER, d1 INTEGER, d0 INTEGER,
+
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_booking_pace_arrival ON newbook_booking_pace(arrival_date);
+
+-- ============================================
+-- PICKUP-V2: CATEGORY BOOKING PACE (per-category room counts at lead times)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS category_booking_pace (
+    id SERIAL PRIMARY KEY,
+    arrival_date DATE NOT NULL,
+    category_id VARCHAR(50) NOT NULL,
+
+    -- Monthly intervals: months 7-12 (6 columns)
+    d365 INTEGER, d330 INTEGER, d300 INTEGER, d270 INTEGER, d240 INTEGER, d210 INTEGER,
+
+    -- Weekly intervals: weeks 5-25 (21 columns)
+    d177 INTEGER, d170 INTEGER, d163 INTEGER, d156 INTEGER, d149 INTEGER,
+    d142 INTEGER, d135 INTEGER, d128 INTEGER, d121 INTEGER, d114 INTEGER,
+    d107 INTEGER, d100 INTEGER, d93 INTEGER, d86 INTEGER, d79 INTEGER,
+    d72 INTEGER, d65 INTEGER, d58 INTEGER, d51 INTEGER, d44 INTEGER, d37 INTEGER,
+
+    -- Daily intervals: days 0-30 (31 columns)
+    d30 INTEGER, d29 INTEGER, d28 INTEGER, d27 INTEGER, d26 INTEGER,
+    d25 INTEGER, d24 INTEGER, d23 INTEGER, d22 INTEGER, d21 INTEGER,
+    d20 INTEGER, d19 INTEGER, d18 INTEGER, d17 INTEGER, d16 INTEGER,
+    d15 INTEGER, d14 INTEGER, d13 INTEGER, d12 INTEGER, d11 INTEGER,
+    d10 INTEGER, d9 INTEGER, d8 INTEGER, d7 INTEGER, d6 INTEGER,
+    d5 INTEGER, d4 INTEGER, d3 INTEGER, d2 INTEGER, d1 INTEGER, d0 INTEGER,
+
+    updated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(arrival_date, category_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_category_booking_pace_arrival ON category_booking_pace(arrival_date);
+CREATE INDEX IF NOT EXISTS idx_category_booking_pace_category ON category_booking_pace(category_id);
+
+-- ============================================
+-- PICKUP-V2: REVENUE PACE (booked accommodation revenue at lead times)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS revenue_pace (
+    id SERIAL PRIMARY KEY,
+    stay_date DATE NOT NULL UNIQUE,
+
+    -- Monthly intervals: months 7-12 (6 columns) - DECIMAL for revenue
+    d365 DECIMAL(12,2), d330 DECIMAL(12,2), d300 DECIMAL(12,2),
+    d270 DECIMAL(12,2), d240 DECIMAL(12,2), d210 DECIMAL(12,2),
+
+    -- Weekly intervals: weeks 5-25 (21 columns)
+    d177 DECIMAL(12,2), d170 DECIMAL(12,2), d163 DECIMAL(12,2), d156 DECIMAL(12,2), d149 DECIMAL(12,2),
+    d142 DECIMAL(12,2), d135 DECIMAL(12,2), d128 DECIMAL(12,2), d121 DECIMAL(12,2), d114 DECIMAL(12,2),
+    d107 DECIMAL(12,2), d100 DECIMAL(12,2), d93 DECIMAL(12,2), d86 DECIMAL(12,2), d79 DECIMAL(12,2),
+    d72 DECIMAL(12,2), d65 DECIMAL(12,2), d58 DECIMAL(12,2), d51 DECIMAL(12,2), d44 DECIMAL(12,2), d37 DECIMAL(12,2),
+
+    -- Daily intervals: days 0-30 (31 columns)
+    d30 DECIMAL(12,2), d29 DECIMAL(12,2), d28 DECIMAL(12,2), d27 DECIMAL(12,2), d26 DECIMAL(12,2),
+    d25 DECIMAL(12,2), d24 DECIMAL(12,2), d23 DECIMAL(12,2), d22 DECIMAL(12,2), d21 DECIMAL(12,2),
+    d20 DECIMAL(12,2), d19 DECIMAL(12,2), d18 DECIMAL(12,2), d17 DECIMAL(12,2), d16 DECIMAL(12,2),
+    d15 DECIMAL(12,2), d14 DECIMAL(12,2), d13 DECIMAL(12,2), d12 DECIMAL(12,2), d11 DECIMAL(12,2),
+    d10 DECIMAL(12,2), d9 DECIMAL(12,2), d8 DECIMAL(12,2), d7 DECIMAL(12,2), d6 DECIMAL(12,2),
+    d5 DECIMAL(12,2), d4 DECIMAL(12,2), d3 DECIMAL(12,2), d2 DECIMAL(12,2), d1 DECIMAL(12,2), d0 DECIMAL(12,2),
+
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_revenue_pace_stay ON revenue_pace(stay_date);
+
+-- ============================================
+-- PICKUP-V2: CURRENT RATES FROM NEWBOOK (for ceiling calculations)
+-- Now with rate history tracking - stores snapshots when rates change
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS newbook_current_rates (
+    id SERIAL PRIMARY KEY,
+    category_id VARCHAR(50) NOT NULL,
+    rate_date DATE NOT NULL,
+    rate_name VARCHAR(255),
+    rate_gross DECIMAL(12,2),
+    rate_net DECIMAL(12,2),
+    tariffs_data JSONB DEFAULT '{}',  -- All available tariff options with availability status
+    valid_from TIMESTAMP DEFAULT NOW(),  -- When this rate version started
+    last_verified_at TIMESTAMP DEFAULT NOW()  -- Last time we confirmed rate is still current
+    -- No UNIQUE constraint - allows multiple versions per (category_id, rate_date)
+);
+
+CREATE INDEX IF NOT EXISTS idx_current_rates_date ON newbook_current_rates(rate_date);
+CREATE INDEX IF NOT EXISTS idx_current_rates_category ON newbook_current_rates(category_id);
+CREATE INDEX IF NOT EXISTS idx_current_rates_tariffs ON newbook_current_rates USING gin(tariffs_data);
+CREATE INDEX IF NOT EXISTS idx_current_rates_latest ON newbook_current_rates(category_id, rate_date, valid_from DESC);
+
+-- Migration: Add tariffs_data column if missing (for existing databases)
+ALTER TABLE newbook_current_rates ADD COLUMN IF NOT EXISTS tariffs_data JSONB DEFAULT '{}';
+
+-- Migration: Convert from old schema to new snapshot schema
+-- Rename fetched_at to valid_from if it exists
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns
+               WHERE table_name = 'newbook_current_rates' AND column_name = 'fetched_at') THEN
+        ALTER TABLE newbook_current_rates RENAME COLUMN fetched_at TO valid_from;
+    END IF;
+END $$;
+
+-- Add last_verified_at column if missing
+ALTER TABLE newbook_current_rates ADD COLUMN IF NOT EXISTS last_verified_at TIMESTAMP DEFAULT NOW();
+
+-- Drop unique constraint if it exists (allows rate history)
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM pg_constraint
+               WHERE conname = 'newbook_current_rates_category_id_rate_date_key') THEN
+        ALTER TABLE newbook_current_rates
+        DROP CONSTRAINT newbook_current_rates_category_id_rate_date_key;
+    END IF;
+END $$;
+
+-- ============================================
+-- FORECASTING TABLES (for Prophet, XGBoost, CatBoost models)
+-- ============================================
+
+-- Forecast metrics configuration
+CREATE TABLE IF NOT EXISTS forecast_metrics (
+    id SERIAL PRIMARY KEY,
+    metric_code VARCHAR(50) NOT NULL UNIQUE,
+    metric_name VARCHAR(100) NOT NULL,
+    description TEXT,
+    unit VARCHAR(20),
+    is_active BOOLEAN DEFAULT TRUE,
+    use_prophet BOOLEAN DEFAULT TRUE,
+    use_xgboost BOOLEAN DEFAULT TRUE,
+    use_pickup BOOLEAN DEFAULT FALSE,
+    use_catboost BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+-- Default metrics for forecasting
+INSERT INTO forecast_metrics (metric_code, metric_name, description, unit, use_prophet, use_xgboost, use_pickup, use_catboost) VALUES
+('hotel_occupancy_pct', 'Hotel Occupancy %', 'Percentage of available rooms occupied', '%', TRUE, TRUE, TRUE, TRUE),
+('hotel_room_nights', 'Room Nights', 'Number of rooms sold', 'rooms', TRUE, TRUE, TRUE, TRUE),
+('hotel_guests', 'Guest Count', 'Total guests staying', 'guests', TRUE, TRUE, FALSE, TRUE),
+('hotel_arrivals', 'Arrivals', 'Number of check-ins', 'arrivals', TRUE, TRUE, FALSE, TRUE)
+ON CONFLICT (metric_code) DO NOTHING;
+
+-- Daily metrics (actuals storage - populated from newbook_bookings_stats)
+CREATE TABLE IF NOT EXISTS daily_metrics (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    metric_code VARCHAR(50) NOT NULL,
+    actual_value DECIMAL(12,2),
+    source VARCHAR(50) DEFAULT 'newbook',
+    updated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, metric_code)
+);
+
+CREATE INDEX IF NOT EXISTS idx_daily_metrics_date ON daily_metrics(date);
+CREATE INDEX IF NOT EXISTS idx_daily_metrics_code ON daily_metrics(metric_code, date);
+
+-- Forecasts storage
+CREATE TABLE IF NOT EXISTS forecasts (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    model_type VARCHAR(20) NOT NULL,  -- 'prophet', 'xgboost', 'pickup', 'catboost'
+    predicted_value DECIMAL(12,2) NOT NULL,
+    lower_bound DECIMAL(12,2),
+    upper_bound DECIMAL(12,2),
+    generated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(forecast_date, forecast_type, model_type, generated_at)
+);
+
+CREATE INDEX IF NOT EXISTS idx_forecasts_date ON forecasts(forecast_date);
+CREATE INDEX IF NOT EXISTS idx_forecasts_type ON forecasts(forecast_type, model_type);
+CREATE INDEX IF NOT EXISTS idx_forecasts_generated ON forecasts(generated_at DESC);
+
+-- Actual vs forecast comparison
+CREATE TABLE IF NOT EXISTS actual_vs_forecast (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    metric_type VARCHAR(50) NOT NULL,
+    actual_value DECIMAL(12,2),
+    budget_value DECIMAL(12,2),
+    -- Prophet
+    prophet_forecast DECIMAL(12,2),
+    prophet_lower DECIMAL(12,2),
+    prophet_upper DECIMAL(12,2),
+    prophet_error DECIMAL(12,4),
+    prophet_pct_error DECIMAL(8,4),
+    -- XGBoost
+    xgboost_forecast DECIMAL(12,2),
+    xgboost_error DECIMAL(12,4),
+    xgboost_pct_error DECIMAL(8,4),
+    -- Pickup
+    pickup_forecast DECIMAL(12,2),
+    pickup_error DECIMAL(12,4),
+    pickup_pct_error DECIMAL(8,4),
+    -- CatBoost
+    catboost_forecast DECIMAL(12,2),
+    catboost_error DECIMAL(12,4),
+    catboost_pct_error DECIMAL(8,4),
+    -- Analysis
+    best_model VARCHAR(20),
+    calculated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(date, metric_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_actual_vs_forecast_date ON actual_vs_forecast(date);
+CREATE INDEX IF NOT EXISTS idx_actual_vs_forecast_type ON actual_vs_forecast(metric_type);
+
+-- Prophet decomposition storage
+CREATE TABLE IF NOT EXISTS prophet_decomposition (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    trend DECIMAL(12,2),
+    yearly_seasonality DECIMAL(12,2),
+    weekly_seasonality DECIMAL(12,2),
+    daily_seasonality DECIMAL(12,2),
+    holiday_effects JSONB,
+    regressor_effects JSONB,
+    generated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_prophet_decomposition_date ON prophet_decomposition(forecast_date, forecast_type);
+
+-- XGBoost SHAP explanations
+CREATE TABLE IF NOT EXISTS xgboost_explanations (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    base_value DECIMAL(12,2),
+    feature_values JSONB,
+    shap_values JSONB,
+    top_positive JSONB,
+    top_negative JSONB,
+    generated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_xgboost_explanations_date ON xgboost_explanations(forecast_date, forecast_type);
+
+-- Pickup model explanations
+CREATE TABLE IF NOT EXISTS pickup_explanations (
+    id SERIAL PRIMARY KEY,
+    run_id UUID,
+    forecast_date DATE NOT NULL,
+    forecast_type VARCHAR(50) NOT NULL,
+    current_otb DECIMAL(12,2),
+    days_out INTEGER,
+    comparison_date DATE,
+    comparison_otb DECIMAL(12,2),
+    comparison_final DECIMAL(12,2),
+    pickup_curve_pct DECIMAL(8,4),
+    pickup_curve_stddev DECIMAL(8,4),
+    pace_vs_prior_pct DECIMAL(8,4),
+    projection_method VARCHAR(50),
+    projected_value DECIMAL(12,2),
+    confidence_note TEXT,
+    generated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_pickup_explanations_date ON pickup_explanations(forecast_date, forecast_type);
+
+-- Monthly budgets from FD
+CREATE TABLE IF NOT EXISTS monthly_budgets (
+    id SERIAL PRIMARY KEY,
+    year INTEGER NOT NULL,
+    month INTEGER NOT NULL CHECK (month >= 1 AND month <= 12),
+    budget_type VARCHAR(50) NOT NULL,
+    budget_value DECIMAL(12,2) NOT NULL,
+    notes TEXT,
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(year, month, budget_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_monthly_budgets_year ON monthly_budgets(year, budget_type);
+
+-- Daily budgets (distributed from monthly)
+CREATE TABLE IF NOT EXISTS daily_budgets (
+    id SERIAL PRIMARY KEY,
+    date DATE NOT NULL,
+    budget_type VARCHAR(50) NOT NULL,
+    budget_value DECIMAL(12,2),
+    distribution_method VARCHAR(50),
+    prior_year_pct DECIMAL(10,6),
+    monthly_budget_id INTEGER REFERENCES monthly_budgets(id),
+    calculated_at TIMESTAMP,
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP,
+    UNIQUE(date, budget_type)
+);
+
+CREATE INDEX IF NOT EXISTS idx_daily_budgets_date ON daily_budgets(date, budget_type);
+
+-- Forecast snapshots (for tracking how forecasts evolve over time)
+CREATE TABLE IF NOT EXISTS forecast_snapshots (
+    id SERIAL PRIMARY KEY,
+    snapshot_date DATE NOT NULL,
+    target_date DATE NOT NULL,
+    metric_code VARCHAR(50) NOT NULL,
+    days_out INTEGER NOT NULL,
+    prophet_value DECIMAL(12,2),
+    xgboost_value DECIMAL(12,2),
+    pickup_value DECIMAL(12,2),
+    catboost_value DECIMAL(12,2),
+    created_at TIMESTAMP DEFAULT NOW(),
+    UNIQUE(snapshot_date, target_date, metric_code)
+);
+
+CREATE INDEX IF NOT EXISTS idx_forecast_snapshots_target ON forecast_snapshots(target_date, metric_code);
+
+-- ============================================
+-- USER ROLES (migration for existing users table)
+-- ============================================
+
+ALTER TABLE users ADD COLUMN IF NOT EXISTS role VARCHAR(20) DEFAULT 'admin';
+
+-- ============================================
+-- RECONCILIATION: CASH UP SESSIONS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_cash_ups (
+    id SERIAL PRIMARY KEY,
+    session_date DATE NOT NULL UNIQUE,
+    created_by INTEGER NOT NULL REFERENCES users(id),
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW(),
+    status VARCHAR(20) DEFAULT 'draft' CHECK (status IN ('draft', 'final')),
+    total_float_counted DECIMAL(10,2) DEFAULT 0.00,
+    total_cash_counted DECIMAL(10,2) DEFAULT 0.00,
+    notes TEXT,
+    submitted_at TIMESTAMP,
+    submitted_by INTEGER REFERENCES users(id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_cash_ups_date ON recon_cash_ups(session_date);
+CREATE INDEX IF NOT EXISTS idx_recon_cash_ups_status ON recon_cash_ups(status);
+
+-- ============================================
+-- RECONCILIATION: DENOMINATION COUNTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_denominations (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    count_type VARCHAR(20) NOT NULL DEFAULT 'takings' CHECK (count_type IN ('float', 'takings')),
+    denomination_type VARCHAR(20) NOT NULL CHECK (denomination_type IN ('note', 'coin')),
+    denomination_value DECIMAL(10,2) NOT NULL,
+    quantity INTEGER DEFAULT NULL,
+    value_entered DECIMAL(10,2) DEFAULT NULL,
+    total_amount DECIMAL(10,2) NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_denoms_cashup ON recon_denominations(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: CARD MACHINE DATA
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_card_machines (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    machine_name VARCHAR(100) NOT NULL,
+    total_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    amex_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    visa_mc_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_cards_cashup ON recon_card_machines(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: NEWBOOK PAYMENT RECORDS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_payment_records (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER REFERENCES recon_cash_ups(id) ON DELETE SET NULL,
+    newbook_payment_id VARCHAR(100),
+    booking_id VARCHAR(100),
+    guest_name VARCHAR(255),
+    payment_date TIMESTAMP NOT NULL,
+    payment_type VARCHAR(100),
+    payment_method VARCHAR(50),
+    transaction_method VARCHAR(50),
+    card_type VARCHAR(50),
+    amount DECIMAL(10,2) NOT NULL,
+    tendered DECIMAL(10,2),
+    processed_by VARCHAR(255),
+    item_type VARCHAR(50),
+    synced_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_payments_cashup ON recon_payment_records(cash_up_id);
+CREATE INDEX IF NOT EXISTS idx_recon_payments_date ON recon_payment_records(payment_date);
+
+-- ============================================
+-- RECONCILIATION: RECONCILIATION RESULTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_reconciliation (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    category VARCHAR(50) NOT NULL,
+    banked_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    reported_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00,
+    variance DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_recon_cashup ON recon_reconciliation(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: DAILY STATISTICS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_daily_stats (
+    id SERIAL PRIMARY KEY,
+    business_date DATE NOT NULL UNIQUE,
+    gross_sales DECIMAL(10,2) DEFAULT 0.00,
+    debtors_creditors_balance DECIMAL(10,2) DEFAULT 0.00,
+    rooms_sold INTEGER DEFAULT 0,
+    total_people INTEGER DEFAULT 0,
+    source VARCHAR(50) DEFAULT 'manual',
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_daily_date ON recon_daily_stats(business_date);
+
+-- ============================================
+-- RECONCILIATION: SALES BREAKDOWN
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_sales_breakdown (
+    id SERIAL PRIMARY KEY,
+    business_date DATE NOT NULL,
+    category VARCHAR(100) NOT NULL,
+    net_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_sales_date ON recon_sales_breakdown(business_date);
+
+-- ============================================
+-- RECONCILIATION: FLOAT COUNTS (Petty Cash, Change Tin, Safe Cash)
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_float_counts (
+    id SERIAL PRIMARY KEY,
+    count_type VARCHAR(20) NOT NULL CHECK (count_type IN ('petty_cash', 'change_tin', 'safe_cash')),
+    count_date TIMESTAMP NOT NULL,
+    created_by INTEGER NOT NULL REFERENCES users(id),
+    created_at TIMESTAMP DEFAULT NOW(),
+    total_counted DECIMAL(10,2) DEFAULT 0.00,
+    total_receipts DECIMAL(10,2) DEFAULT 0.00,
+    target_amount DECIMAL(10,2) DEFAULT 0.00,
+    variance DECIMAL(10,2) DEFAULT 0.00,
+    notes TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_floats_type ON recon_float_counts(count_type);
+CREATE INDEX IF NOT EXISTS idx_recon_floats_date ON recon_float_counts(count_date);
+
+-- ============================================
+-- RECONCILIATION: FLOAT DENOMINATION COUNTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_float_denominations (
+    id SERIAL PRIMARY KEY,
+    float_count_id INTEGER NOT NULL REFERENCES recon_float_counts(id) ON DELETE CASCADE,
+    denomination_value DECIMAL(10,2) NOT NULL,
+    quantity INTEGER DEFAULT 0,
+    total_amount DECIMAL(10,2) NOT NULL DEFAULT 0.00
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_float_denoms_fc ON recon_float_denominations(float_count_id);
+
+-- ============================================
+-- RECONCILIATION: FLOAT RECEIPTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_float_receipts (
+    id SERIAL PRIMARY KEY,
+    float_count_id INTEGER NOT NULL REFERENCES recon_float_counts(id) ON DELETE CASCADE,
+    receipt_value DECIMAL(10,2) NOT NULL,
+    receipt_description VARCHAR(255)
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_float_receipts_fc ON recon_float_receipts(float_count_id);
+
+-- ============================================
+-- RECONCILIATION: CASH COUNT ATTACHMENTS
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS recon_attachments (
+    id SERIAL PRIMARY KEY,
+    cash_up_id INTEGER NOT NULL REFERENCES recon_cash_ups(id) ON DELETE CASCADE,
+    file_name VARCHAR(255) NOT NULL,
+    file_path VARCHAR(500) NOT NULL,
+    file_type VARCHAR(50) NOT NULL,
+    file_size BIGINT NOT NULL,
+    uploaded_by INTEGER NOT NULL REFERENCES users(id),
+    uploaded_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_recon_attachments_cashup ON recon_attachments(cash_up_id);
+
+-- ============================================
+-- RECONCILIATION: SYSTEM CONFIG ENTRIES
+-- ============================================
+
+INSERT INTO system_config (config_key, config_value, description) VALUES
+('recon_expected_till_float', '300.00', 'Expected till float amount (GBP)'),
+('recon_variance_threshold', '10.00', 'Variance threshold for highlighting (GBP)'),
+('recon_default_report_days', '7', 'Default number of days for multi-day reports'),
+('recon_petty_cash_target', '200.00', 'Target amount for petty cash float'),
+('recon_change_tin_breakdown', '{"50.00":0,"20.00":0,"10.00":0,"5.00":0,"2.00":20.00,"1.00":20.00,"0.50":10.00,"0.20":10.00,"0.10":5.00,"0.05":5.00}', 'Change tin denomination breakdown targets (JSON)'),
+('recon_denominations', '{"notes":[50.00,20.00,10.00,5.00],"coins":[2.00,1.00,0.50,0.20,0.10,0.05,0.02,0.01]}', 'GBP denominations configuration (JSON)'),
+('recon_sales_breakdown_columns', '[]', 'Sales breakdown column configuration (JSON)'),
+('recon_safe_cash_target', '0.00', 'Target amount for safe cash float')
+ON CONFLICT (config_key) DO NOTHING;

docker-compose.yml

@@ -0,0 +1,45 @@
+services:
+  backend:
+    build: ./backend
+    security_opt:
+      - apparmor=unconfined
+    environment:
+      - DATABASE_URL=${DATABASE_URL}
+      - CENTRAL_AUTH_SECRET=${CENTRAL_AUTH_SECRET}
+      - APP_SLUG=forecasting
+      - NEWBOOK_API_KEY=${NEWBOOK_API_KEY:-}
+      - NEWBOOK_USERNAME=${NEWBOOK_USERNAME:-}
+      - NEWBOOK_PASSWORD=${NEWBOOK_PASSWORD:-}
+      - NEWBOOK_REGION=${NEWBOOK_REGION:-AU}
+      - RESOS_API_KEY=${RESOS_API_KEY:-}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+    volumes:
+      - recon_uploads:/app/uploads/reconciliation
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 15s
+      timeout: 10s
+      retries: 10
+      start_period: 60s
+    restart: unless-stopped
+
+  frontend:
+    build:
+      context: ./frontend
+      args:
+        VITE_HOTEL_NAME: ${VITE_HOTEL_NAME:-Hotel}
+    security_opt:
+      - apparmor=unconfined
+    ports:
+      - "${FRONTEND_PORT:-3080}:80"
+    depends_on:
+      backend:
+        condition: service_healthy
+    restart: unless-stopped
+
+networks:
+  default:
+    driver: bridge
+
+volumes:
+  recon_uploads:

frontend/Dockerfile

@@ -0,0 +1,15 @@
+FROM node:22-alpine AS builder
+
+ARG VITE_HOTEL_NAME=Hotel
+ENV VITE_HOTEL_NAME=$VITE_HOTEL_NAME
+
+WORKDIR /app
+COPY package.json ./
+RUN npm install
+COPY . .
+RUN npm run build
+
+FROM nginx:alpine
+COPY --from=builder /app/dist /usr/share/nginx/html/forecasting
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+EXPOSE 80

frontend/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/forecasting/favicon.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Forecasting</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

frontend/nginx.conf

@@ -0,0 +1,35 @@
+server {
+  listen 80;
+  server_name _;
+
+  # 1. Central auth proxy
+  location /forecasting/api/auth/ {
+    proxy_pass http://10.10.10.101:3001/api/auth/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  }
+
+  # 2. App backend (Python FastAPI on port 8000)
+  location /forecasting/api/ {
+    proxy_pass http://backend:8000/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Cookie $http_cookie;
+    proxy_read_timeout 300s;
+    proxy_connect_timeout 30s;
+    client_max_body_size 50M;
+  }
+
+  # 3. Health
+  location /forecasting/health {
+    proxy_pass http://backend:8000/health;
+  }
+
+  # 4. SPA fallback
+  location /forecasting/ {
+    root /usr/share/nginx/html;
+    try_files $uri $uri/ /forecasting/index.html;
+  }
+}

frontend/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "forecasting-frontend",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@tanstack/react-query": "^5.32.0",
+    "axios": "^1.6.8",
+    "lucide-react": "^0.395.0",
+    "plotly.js": "^2.29.0",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-plotly.js": "^2.6.0",
+    "react-router-dom": "^6.22.0"
+  },
+  "devDependencies": {
+    "@types/plotly.js": "^2.12.29",
+    "@types/react": "^18.3.1",
+    "@types/react-dom": "^18.3.0",
+    "@types/react-plotly.js": "^2.6.4",
+    "@vitejs/plugin-react": "^4.2.1",
+    "typescript": "^5.4.5",
+    "vite": "^5.2.11"
+  }
+}

frontend/src/App.tsx

@@ -0,0 +1,34 @@
+import { Routes, Route, Navigate } from 'react-router-dom'
+import AuthGate from './components/AuthGate'
+import Layout from './components/Layout'
+import Dashboard from './pages/Dashboard'
+import Forecasts from './pages/Forecasts'
+import History from './pages/History'
+import Bookability from './pages/Bookability'
+import CompetitorRates from './pages/CompetitorRates'
+import Accuracy from './pages/Accuracy'
+import Settings from './pages/Settings'
+
+export default function App() {
+  return (
+    <AuthGate>
+      <Layout>
+        <Routes>
+          <Route path="/" element={<Navigate to="/dashboard" replace />} />
+          <Route path="/dashboard" element={<Dashboard />} />
+          <Route path="/forecasts" element={<Forecasts />} />
+          <Route path="/forecasts/:page" element={<Forecasts />} />
+          <Route path="/history" element={<History />} />
+          <Route path="/history/:report" element={<History />} />
+          <Route path="/bookability" element={<Bookability />} />
+          <Route path="/competitor-rates" element={<CompetitorRates />} />
+          <Route path="/accuracy" element={<Accuracy />} />
+          <Route path="/accuracy/:tab" element={<Accuracy />} />
+          <Route path="/settings" element={<Settings />} />
+          <Route path="/settings/:tab" element={<Settings />} />
+          <Route path="*" element={<Navigate to="/dashboard" replace />} />
+        </Routes>
+      </Layout>
+    </AuthGate>
+  )
+}

frontend/src/api.ts

@@ -0,0 +1,20 @@
+import axios from 'axios'
+
+const BASE = '/forecasting/api'
+
+const api = axios.create({
+  baseURL: BASE,
+  withCredentials: true,
+})
+
+api.interceptors.response.use(
+  (response) => response,
+  (error) => {
+    if (error.response?.status === 401) {
+      window.location.href = '/auth/login?redirect=' + encodeURIComponent(window.location.pathname)
+    }
+    return Promise.reject(error)
+  }
+)
+
+export default api

frontend/src/components/AuthGate.tsx

@@ -0,0 +1,47 @@
+import { createContext, useContext, useEffect, useState } from 'react'
+import type { ReactNode } from 'react'
+import type { User } from '../types'
+
+interface AuthCtx { user: User }
+const Ctx = createContext<AuthCtx | null>(null)
+
+export function useAuth() {
+  const ctx = useContext(Ctx)
+  if (!ctx) throw new Error('useAuth outside AuthGate')
+  return ctx
+}
+
+export default function AuthGate({ children }: { children: ReactNode }) {
+  const [user, setUser] = useState<User | null>(null)
+  const [checking, setChecking] = useState(true)
+
+  useEffect(() => {
+    fetch('/forecasting/api/auth/verify?app=forecasting', { credentials: 'include' })
+      .then(r => {
+        if (!r.ok) throw new Error('unauth')
+        return r.json()
+      })
+      .then(data => setUser({
+        email: data.email || data.sub || '',
+        name: data.name || data.display_name || '',
+        is_admin: data.is_admin ?? false,
+        caps: data.caps ?? [],
+      }))
+      .catch(() => {
+        window.location.href = '/auth/login?redirect=' + encodeURIComponent(window.location.pathname)
+      })
+      .finally(() => setChecking(false))
+  }, [])
+
+  if (checking) {
+    return (
+      <div style={{ minHeight: '100vh', display: 'flex', alignItems: 'center', justifyContent: 'center', background: 'var(--navy-dark)' }}>
+        <div className="spinner" style={{ width: 32, height: 32 }} />
+      </div>
+    )
+  }
+
+  if (!user) return null
+
+  return <Ctx.Provider value={{ user }}>{children}</Ctx.Provider>
+}

frontend/src/components/Layout.tsx

@@ -0,0 +1,59 @@
+import { NavLink } from 'react-router-dom'
+import {
+  TrendingUp, BarChart2, Calendar, Target, Globe, History,
+  Settings, Bot,
+} from 'lucide-react'
+import { useAuth } from './AuthGate'
+import { can } from '../types'
+import type { ReactNode } from 'react'
+
+const ICON = { size: 16, strokeWidth: 1.75 }
+
+const NAV = [
+  { to: '/dashboard',         label: 'Dashboard',         icon: Bot,       cap: 'view' },
+  { to: '/forecasts',         label: 'Forecasts',         icon: TrendingUp, cap: 'view' },
+  { to: '/history',           label: 'History',           icon: History,   cap: 'view' },
+  { to: '/bookability',       label: 'Bookability',       icon: Calendar,  cap: 'view_bookability' },
+  { to: '/competitor-rates',  label: 'Competitors',       icon: Globe,     cap: 'view_competitor_rates' },
+  { to: '/accuracy',          label: 'Accuracy',          icon: Target,    cap: 'view_accuracy' },
+  { to: '/settings',          label: 'Settings',          icon: Settings,  cap: 'settings' },
+]
+
+export default function Layout({ children }: { children: ReactNode }) {
+  const { user } = useAuth()
+  const items = NAV.filter(n => can(user, n.cap))
+
+  return (
+    <div className="app-shell">
+      <aside className="sidebar">
+        <div className="sidebar-logo">
+          <BarChart2 size={18} strokeWidth={1.75} />
+          Forecasting
+        </div>
+        <nav className="sidebar-nav">
+          {items.map(({ to, label, icon: Icon }) => (
+            <NavLink key={to} to={to} className={({ isActive }) => isActive ? 'active' : ''}>
+              <Icon {...ICON} />
+              {label}
+            </NavLink>
+          ))}
+        </nav>
+        <div className="sidebar-user">{user.name || user.email}</div>
+      </aside>
+
+      <header className="top-bar">
+        <BarChart2 size={18} strokeWidth={1.75} color="var(--gold)" />
+        <span className="top-bar-title">Forecasting</span>
+        <nav className="top-bar-nav">
+          {items.map(({ to, label }) => (
+            <NavLink key={to} to={to} className={({ isActive }) => isActive ? 'active' : ''}>
+              {label}
+            </NavLink>
+          ))}
+        </nav>
+      </header>
+
+      <main className="page-content">{children}</main>
+    </div>
+  )
+}

frontend/src/index.css

@@ -0,0 +1,361 @@
+:root {
+  --navy:        #1a1a2e;
+  --navy-dark:   #0f0f20;
+  --gold:        #c9a84c;
+  --gold-light:  #e8c96d;
+  --surface:     rgba(255,255,255,0.07);
+  --surface-2:   rgba(255,255,255,0.08);
+  --text:        rgba(255,255,255,0.88);
+  --text-muted:  rgba(255,255,255,0.48);
+  --body-bg:     #f4f5f7;
+  --card-bg:     #ffffff;
+  --card-border: #e4e8ee;
+  --text-dark:   #1e293b;
+  --text-mid:    #64748b;
+  --shadow-sm:   0 1px 3px rgba(0,0,0,0.07), 0 1px 2px rgba(0,0,0,0.04);
+  --shadow-md:   0 4px 12px rgba(0,0,0,0.08);
+  --danger:      #dc2626;
+  --success:     #16a34a;
+  --warning:     #d97706;
+  --radius:      10px;
+  --font:        -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+}
+
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+body {
+  background: var(--body-bg);
+  color: var(--text-dark);
+  font-family: var(--font);
+  font-size: 14px;
+  line-height: 1.5;
+}
+
+/* App shell layout */
+.app-shell {
+  display: grid;
+  grid-template-columns: 220px 1fr;
+  grid-template-rows: auto 1fr;
+  min-height: 100vh;
+}
+
+.sidebar {
+  grid-column: 1;
+  grid-row: 1 / -1;
+  background: var(--navy);
+  display: flex;
+  flex-direction: column;
+  padding: 0;
+  position: sticky;
+  top: 0;
+  height: 100vh;
+  overflow-y: auto;
+}
+
+.sidebar-logo {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  padding: 20px 16px 16px;
+  font-size: 15px;
+  font-weight: 600;
+  color: var(--text);
+  border-bottom: 1px solid var(--surface);
+}
+
+.sidebar-nav {
+  flex: 1;
+  padding: 12px 8px;
+  display: flex;
+  flex-direction: column;
+  gap: 2px;
+}
+
+.sidebar-nav a {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  padding: 9px 10px;
+  border-radius: 8px;
+  color: var(--text-muted);
+  text-decoration: none;
+  font-size: 13.5px;
+  transition: background 0.15s, color 0.15s;
+}
+
+.sidebar-nav a:hover { background: var(--surface); color: var(--text); }
+.sidebar-nav a.active { background: rgba(201,168,76,0.15); color: var(--gold); }
+
+.sidebar-user {
+  padding: 12px 16px;
+  font-size: 12px;
+  color: var(--text-muted);
+  border-top: 1px solid var(--surface);
+}
+
+/* Top bar — mobile/collapsed fallback */
+.top-bar {
+  display: none;
+}
+
+.page-content {
+  grid-column: 2;
+  grid-row: 1 / -1;
+  min-width: 0;
+  padding: 24px;
+  background: var(--body-bg);
+}
+
+/* Cards */
+.card {
+  background: var(--card-bg);
+  border: 1px solid var(--card-border);
+  border-radius: var(--radius);
+  box-shadow: var(--shadow-sm);
+}
+
+.card-header {
+  padding: 16px 20px;
+  border-bottom: 1px solid var(--card-border);
+  font-size: 14px;
+  font-weight: 600;
+  color: var(--text-dark);
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+}
+
+.card-body { padding: 20px; }
+
+/* Buttons */
+.btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 7px 14px;
+  border-radius: 7px;
+  font-size: 13px;
+  font-weight: 500;
+  cursor: pointer;
+  border: none;
+  transition: background 0.15s, opacity 0.15s;
+}
+
+.btn:disabled { opacity: 0.5; cursor: not-allowed; }
+
+.btn-primary {
+  background: var(--gold);
+  color: var(--navy);
+}
+.btn-primary:hover:not(:disabled) { background: var(--gold-light); }
+
+.btn-secondary {
+  background: var(--navy);
+  color: var(--text);
+}
+.btn-secondary:hover:not(:disabled) { background: var(--navy-dark); }
+
+.btn-outline {
+  background: transparent;
+  color: var(--text-dark);
+  border: 1px solid var(--card-border);
+}
+.btn-outline:hover:not(:disabled) { background: var(--body-bg); }
+
+.btn-danger {
+  background: var(--danger);
+  color: white;
+}
+.btn-danger:hover:not(:disabled) { opacity: 0.85; }
+
+.btn-sm { padding: 4px 10px; font-size: 12px; }
+
+/* Form elements */
+input, select, textarea {
+  background: var(--card-bg);
+  border: 1px solid var(--card-border);
+  border-radius: 7px;
+  padding: 7px 10px;
+  font-size: 13px;
+  color: var(--text-dark);
+  width: 100%;
+  outline: none;
+  transition: border-color 0.15s;
+}
+input:focus, select:focus, textarea:focus { border-color: var(--gold); }
+
+/* Badges */
+.badge {
+  display: inline-flex;
+  align-items: center;
+  padding: 2px 8px;
+  border-radius: 20px;
+  font-size: 11px;
+  font-weight: 500;
+}
+.badge-success { background: #dcfce7; color: #16a34a; }
+.badge-warning { background: #fef3c7; color: #d97706; }
+.badge-danger  { background: #fee2e2; color: #dc2626; }
+.badge-info    { background: #dbeafe; color: #2563eb; }
+.badge-neutral { background: #f1f5f9; color: #64748b; }
+
+/* Tables */
+.table-wrap { overflow-x: auto; }
+table { width: 100%; border-collapse: collapse; font-size: 13px; }
+th {
+  text-align: left;
+  padding: 8px 12px;
+  background: #f8fafc;
+  border-bottom: 1px solid var(--card-border);
+  font-weight: 600;
+  color: var(--text-mid);
+  white-space: nowrap;
+}
+td {
+  padding: 8px 12px;
+  border-bottom: 1px solid #f1f5f9;
+  color: var(--text-dark);
+}
+tr:last-child td { border-bottom: none; }
+tr:hover td { background: #f8fafc; }
+
+/* Page header */
+.page-header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  margin-bottom: 24px;
+  gap: 16px;
+}
+.page-title {
+  font-size: 20px;
+  font-weight: 700;
+  color: var(--text-dark);
+}
+.page-subtitle { font-size: 13px; color: var(--text-mid); margin-top: 2px; }
+
+/* Sub-nav tabs */
+.sub-nav {
+  display: flex;
+  gap: 4px;
+  border-bottom: 1px solid var(--card-border);
+  margin-bottom: 24px;
+  overflow-x: auto;
+}
+.sub-nav-item {
+  padding: 8px 14px;
+  font-size: 13px;
+  font-weight: 500;
+  color: var(--text-mid);
+  cursor: pointer;
+  border-bottom: 2px solid transparent;
+  white-space: nowrap;
+  background: none;
+  border-left: none;
+  border-right: none;
+  border-top: none;
+  transition: color 0.15s, border-color 0.15s;
+}
+.sub-nav-item:hover { color: var(--text-dark); }
+.sub-nav-item.active { color: var(--gold); border-bottom-color: var(--gold); }
+
+/* Status dot */
+.status-dot {
+  width: 8px; height: 8px;
+  border-radius: 50%;
+  display: inline-block;
+}
+.status-dot.green  { background: var(--success); }
+.status-dot.yellow { background: var(--warning); }
+.status-dot.red    { background: var(--danger); }
+.status-dot.grey   { background: #94a3b8; }
+
+/* Spinner */
+.spinner {
+  width: 20px; height: 20px;
+  border: 2px solid var(--card-border);
+  border-top-color: var(--gold);
+  border-radius: 50%;
+  animation: spin 0.7s linear infinite;
+  display: inline-block;
+}
+@keyframes spin { to { transform: rotate(360deg); } }
+
+/* Loading / empty states */
+.loading-state {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 10px;
+  padding: 60px 20px;
+  color: var(--text-mid);
+  font-size: 13px;
+}
+
+.empty-state {
+  text-align: center;
+  padding: 60px 20px;
+  color: var(--text-mid);
+  font-size: 13px;
+}
+
+/* Grid helpers */
+.grid-2 { display: grid; grid-template-columns: repeat(2, 1fr); gap: 16px; }
+.grid-3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }
+.grid-4 { display: grid; grid-template-columns: repeat(4, 1fr); gap: 16px; }
+
+/* Stat card */
+.stat-card {
+  background: var(--card-bg);
+  border: 1px solid var(--card-border);
+  border-radius: var(--radius);
+  padding: 16px 20px;
+  box-shadow: var(--shadow-sm);
+}
+.stat-label { font-size: 12px; color: var(--text-mid); margin-bottom: 4px; }
+.stat-value { font-size: 24px; font-weight: 700; color: var(--text-dark); }
+.stat-delta { font-size: 12px; margin-top: 4px; }
+.stat-delta.positive { color: var(--success); }
+.stat-delta.negative { color: var(--danger); }
+.stat-delta.neutral  { color: var(--text-mid); }
+
+/* Responsive — at narrow widths hide sidebar, show top-bar */
+@media (max-width: 900px) {
+  .app-shell {
+    grid-template-columns: 1fr;
+    grid-template-rows: auto 1fr;
+  }
+  .sidebar { display: none; }
+  .top-bar {
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    background: var(--navy);
+    padding: 0 16px;
+    height: 52px;
+    grid-column: 1;
+    overflow-x: auto;
+  }
+  .top-bar-title {
+    font-size: 14px;
+    font-weight: 600;
+    color: var(--text);
+    white-space: nowrap;
+    margin-right: 8px;
+  }
+  .top-bar-nav { display: flex; gap: 4px; }
+  .top-bar-nav a {
+    color: var(--text-muted);
+    text-decoration: none;
+    font-size: 12.5px;
+    padding: 6px 10px;
+    border-radius: 6px;
+    white-space: nowrap;
+  }
+  .top-bar-nav a:hover { color: var(--text); background: var(--surface); }
+  .top-bar-nav a.active { color: var(--gold); }
+  .page-content {
+    grid-column: 1;
+    padding: 16px;
+  }
+}

frontend/src/main.tsx

@@ -0,0 +1,20 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import { BrowserRouter } from 'react-router-dom'
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
+import App from './App'
+import './index.css'
+
+const qc = new QueryClient({
+  defaultOptions: { queries: { retry: 1, staleTime: 30_000 } },
+})
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <BrowserRouter basename="/forecasting">
+      <QueryClientProvider client={qc}>
+        <App />
+      </QueryClientProvider>
+    </BrowserRouter>
+  </React.StrictMode>,
+)

frontend/src/pages/Dashboard.tsx

@@ -0,0 +1,133 @@
+import { useState } from 'react'
+import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
+import { RefreshCw, Bot, Clock } from 'lucide-react'
+import api from '../api'
+
+interface AIInsight {
+  id: number
+  generated_at: string
+  content: string
+  model: string
+  input_tokens: number
+  output_tokens: number
+  triggered_by: string
+}
+
+function formatAge(iso: string): string {
+  const ms = Date.now() - new Date(iso).getTime()
+  const mins = Math.floor(ms / 60000)
+  if (mins < 1) return 'just now'
+  if (mins < 60) return `${mins}m ago`
+  const hours = Math.floor(mins / 60)
+  if (hours < 24) return `${hours}h ago`
+  return `${Math.floor(hours / 24)}d ago`
+}
+
+function renderContent(text: string) {
+  return text.split('\n').map((line, i) => {
+    const processed = line.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
+    if (line.startsWith('- ') || line.startsWith('* ')) {
+      return (
+        <div key={i} style={{ display: 'flex', gap: 8, marginBottom: 4 }}>
+          <span style={{ color: 'var(--gold)', flexShrink: 0 }}>•</span>
+          <span dangerouslySetInnerHTML={{ __html: processed.slice(2) }} />
+        </div>
+      )
+    }
+    if (line.startsWith('## ') || line.startsWith('# ')) {
+      const txt = line.replace(/^#+\s*/, '')
+      return <p key={i} style={{ fontWeight: 600, marginTop: 12, marginBottom: 6, color: 'var(--text-dark)' }}>{txt}</p>
+    }
+    if (line.trim() === '') return <div key={i} style={{ height: 8 }} />
+    return <p key={i} style={{ marginBottom: 4 }} dangerouslySetInnerHTML={{ __html: processed }} />
+  })
+}
+
+export default function Dashboard() {
+  const qc = useQueryClient()
+  const [genError, setGenError] = useState<string | null>(null)
+
+  const { data: insight, isLoading } = useQuery<AIInsight | null>({
+    queryKey: ['ai-insights-latest'],
+    queryFn: () => api.get('/ai-insights/latest').then(r => r.data),
+    refetchInterval: 5 * 60_000,
+  })
+
+  const generate = useMutation({
+    mutationFn: () => api.post('/ai-insights/generate').then(r => r.data),
+    onSuccess: () => {
+      setGenError(null)
+      qc.invalidateQueries({ queryKey: ['ai-insights-latest'] })
+    },
+    onError: (err: any) => {
+      setGenError(err.response?.data?.detail || 'Failed to generate insight')
+    },
+  })
+
+  return (
+    <div>
+      <div className="page-header">
+        <div>
+          <div className="page-title">Dashboard</div>
+          <div className="page-subtitle">Daily AI-generated forecast summary</div>
+        </div>
+        <button
+          className="btn btn-primary"
+          onClick={() => generate.mutate()}
+          disabled={generate.isPending}
+        >
+          <RefreshCw size={14} strokeWidth={1.75} />
+          {generate.isPending ? 'Generating…' : 'Generate Now'}
+        </button>
+      </div>
+
+      {genError && (
+        <div style={{ background: '#fee2e2', border: '1px solid #fca5a5', borderRadius: 8, padding: '10px 14px', marginBottom: 16, color: '#dc2626', fontSize: 13 }}>
+          {genError}
+        </div>
+      )}
+
+      <div className="card">
+        <div className="card-header">
+          <span style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
+            <Bot size={16} strokeWidth={1.75} color="var(--gold)" />
+            AI Insight
+          </span>
+          {insight && (
+            <span style={{ display: 'flex', alignItems: 'center', gap: 5, fontSize: 12, color: 'var(--text-mid)', fontWeight: 400 }}>
+              <Clock size={12} strokeWidth={1.75} />
+              {formatAge(insight.generated_at)}
+              {insight.model && <span style={{ marginLeft: 6, background: '#f1f5f9', borderRadius: 4, padding: '1px 6px' }}>{insight.model}</span>}
+            </span>
+          )}
+        </div>
+        <div className="card-body" style={{ fontSize: 13.5, lineHeight: 1.65, color: 'var(--text-dark)' }}>
+          {isLoading && (
+            <div className="loading-state">
+              <div className="spinner" />
+              Loading insight…
+            </div>
+          )}
+          {!isLoading && !insight && (
+            <div className="empty-state">
+              <Bot size={32} strokeWidth={1.75} color="var(--text-mid)" style={{ margin: '0 auto 12px' }} />
+              <p>No insight generated yet.</p>
+              <p style={{ marginTop: 6 }}>Click <strong>Generate Now</strong> to produce a daily summary.</p>
+            </div>
+          )}
+          {insight && (
+            <>
+              <div style={{ marginBottom: 16 }}>{renderContent(insight.content)}</div>
+              {(insight.input_tokens || insight.output_tokens) && (
+                <div style={{ fontSize: 11, color: 'var(--text-mid)', marginTop: 16, paddingTop: 12, borderTop: '1px solid var(--card-border)' }}>
+                  {insight.input_tokens}↑ / {insight.output_tokens}↓ tokens
+                  · {insight.triggered_by}
+                </div>
+              )}
+            </>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}

frontend/src/types.ts

@@ -0,0 +1,11 @@
+export interface User {
+  email: string
+  name: string
+  is_admin: boolean
+  caps: string[]
+}
+
+export function can(user: User | null, cap: string): boolean {
+  if (!user) return false
+  return user.is_admin || user.caps.includes(cap)
+}

frontend/src/vite-env.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="vite/client" />
+
+interface ImportMetaEnv {
+  readonly VITE_HOTEL_NAME: string
+}
+
+interface ImportMeta {
+  readonly env: ImportMetaEnv
+}