hotel-manage-stack/forecasting
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Forecasting app: hybrid port to HNF stack

Browse source 
Python FastAPI ML backend kept intact; auth replaced with central hnf_session cookie verification. Frontend rebuilt on React 18 + TS + Vite with stack design system, Plotly charts retained. Shared Postgres via DATABASE_URL; schema applied on startup.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-04 18:49:34 +00:00
commit 75d2c1fa9d
103 changed files with 70316 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
backend/jobs/__init__.py Normal file
View file

@ -0,0 +1 @@
# Scheduled jobs

182
backend/jobs/accuracy_calc.py Normal file
View file

@ -0,0 +1,182 @@
"""
Accuracy calculation job
Compares forecasts to actuals once dates have passed
"""
import logging
from datetime import date, timedelta
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
async def run_accuracy_calculation():
"""
Calculate forecast accuracy for dates that have passed.
Updates actual_vs_forecast table with error metrics.
"""
logger.info("Starting accuracy calculation")
db = next(iter([SyncSessionLocal()]))
try:
# Process yesterday's actuals
calc_date = date.today() - timedelta(days=1)
# Get all metrics
metrics_result = db.execute(
text("""
SELECT metric_code FROM forecast_metrics WHERE is_active = TRUE
""")
)
metrics = [row.metric_code for row in metrics_result.fetchall()]
for metric_code in metrics:
# Get actual value from daily_metrics
actual_result = db.execute(
text("""
SELECT actual_value FROM daily_metrics
WHERE date = :calc_date AND metric_code = :metric_code
"""),
{"calc_date": calc_date, "metric_code": metric_code}
)
actual_row = actual_result.fetchone()
actual_value = actual_row.actual_value if actual_row else None
if actual_value is None:
continue # Skip if no actual available
# Get forecasts for this date
forecast_result = db.execute(
text("""
SELECT model_type, predicted_value, lower_bound, upper_bound
FROM forecasts
WHERE forecast_date = :calc_date AND forecast_type = :metric_code
"""),
{"calc_date": calc_date, "metric_code": metric_code}
)
forecasts = {row.model_type: row for row in forecast_result.fetchall()}
prophet_forecast = forecasts.get('prophet')
xgboost_forecast = forecasts.get('xgboost')
pickup_forecast = forecasts.get('pickup')
catboost_forecast = forecasts.get('catboost')
# Calculate errors
def calc_error(forecast_val):
if forecast_val is None:
return None, None
error = actual_value - forecast_val
pct_error = (error / actual_value * 100) if actual_value != 0 else None
return error, pct_error
prophet_error, prophet_pct = calc_error(
prophet_forecast.predicted_value if prophet_forecast else None
)
xgboost_error, xgboost_pct = calc_error(
xgboost_forecast.predicted_value if xgboost_forecast else None
)
pickup_error, pickup_pct = calc_error(
pickup_forecast.predicted_value if pickup_forecast else None
)
catboost_error, catboost_pct = calc_error(
catboost_forecast.predicted_value if catboost_forecast else None
)
# Determine best model
errors = []
if prophet_error is not None:
errors.append(('prophet', abs(prophet_error)))
if xgboost_error is not None:
errors.append(('xgboost', abs(xgboost_error)))
if pickup_error is not None:
errors.append(('pickup', abs(pickup_error)))
if catboost_error is not None:
errors.append(('catboost', abs(catboost_error)))
best_model = min(errors, key=lambda x: x[1])[0] if errors else None
# Get budget value
budget_result = db.execute(
text("""
SELECT budget_value FROM daily_budgets
WHERE date = :calc_date AND budget_type = :metric_code
"""),
{"calc_date": calc_date, "metric_code": metric_code}
)
budget_row = budget_result.fetchone()
budget_value = budget_row.budget_value if budget_row else None
# Upsert accuracy record
db.execute(
text("""
INSERT INTO actual_vs_forecast (
date, metric_type, actual_value,
prophet_forecast, prophet_lower, prophet_upper,
xgboost_forecast, pickup_forecast,
catboost_forecast,
budget_value,
prophet_error, prophet_pct_error,
xgboost_error, xgboost_pct_error,
pickup_error, pickup_pct_error,
catboost_error, catboost_pct_error,
best_model, calculated_at
) VALUES (
:date, :metric_type, :actual,
:prophet_val, :prophet_lower, :prophet_upper,
:xgboost_val, :pickup_val,
:catboost_val,
:budget,
:prophet_error, :prophet_pct,
:xgboost_error, :xgboost_pct,
:pickup_error, :pickup_pct,
:catboost_error, :catboost_pct,
:best_model, NOW()
)
ON CONFLICT (date, metric_type) DO UPDATE SET
actual_value = :actual,
prophet_error = :prophet_error,
prophet_pct_error = :prophet_pct,
xgboost_error = :xgboost_error,
xgboost_pct_error = :xgboost_pct,
pickup_error = :pickup_error,
pickup_pct_error = :pickup_pct,
catboost_forecast = :catboost_val,
catboost_error = :catboost_error,
catboost_pct_error = :catboost_pct,
best_model = :best_model,
calculated_at = NOW()
"""),
{
"date": calc_date,
"metric_type": metric_code,
"actual": actual_value,
"prophet_val": prophet_forecast.predicted_value if prophet_forecast else None,
"prophet_lower": prophet_forecast.lower_bound if prophet_forecast else None,
"prophet_upper": prophet_forecast.upper_bound if prophet_forecast else None,
"xgboost_val": xgboost_forecast.predicted_value if xgboost_forecast else None,
"pickup_val": pickup_forecast.predicted_value if pickup_forecast else None,
"catboost_val": catboost_forecast.predicted_value if catboost_forecast else None,
"budget": budget_value,
"prophet_error": prophet_error,
"prophet_pct": prophet_pct,
"xgboost_error": xgboost_error,
"xgboost_pct": xgboost_pct,
"pickup_error": pickup_error,
"pickup_pct": pickup_pct,
"catboost_error": catboost_error,
"catboost_pct": catboost_pct,
"best_model": best_model
}
)
db.commit()
logger.info(f"Accuracy calculation completed for {calc_date}")
except Exception as e:
logger.error(f"Accuracy calculation failed: {e}")
db.rollback()
raise
finally:
db.close()

765
backend/jobs/aggregation.py Normal file
View file

@ -0,0 +1,765 @@
"""
Aggregation job - calculates daily summaries from raw booking data
Processes dates from aggregation_queue and updates:
- daily_occupancy (from newbook_bookings)
- daily_covers (from resos_bookings)
"""
import json
import logging
from datetime import date, datetime
from typing import List, Optional
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
async def run_aggregation(source: Optional[str] = None):
"""
Process pending aggregation queue and update daily summary tables.
Args:
source: Optional filter - 'newbook' or 'resos'. If None, processes both.
"""
logger.info(f"Starting aggregation job (source={source or 'all'})")
db = next(iter([SyncSessionLocal()]))
try:
# Get pending dates from queue
query = """
SELECT DISTINCT date, source
FROM aggregation_queue
WHERE aggregated_at IS NULL
"""
if source:
query += " AND source = :source"
query += " ORDER BY date"
result = db.execute(text(query), {"source": source} if source else {})
pending = result.fetchall()
if not pending:
logger.info("No pending dates to aggregate")
return
logger.info(f"Found {len(pending)} date/source combinations to aggregate")
# Group by source
newbook_dates = [row.date for row in pending if row.source == 'newbook']
resos_dates = [row.date for row in pending if row.source == 'resos']
# Process Newbook dates
if newbook_dates:
await aggregate_newbook_dates(db, newbook_dates)
# Process Resos dates
if resos_dates:
await aggregate_resos_dates(db, resos_dates)
# Populate daily_metrics from aggregated data (for forecasting models)
all_dates = list(set(newbook_dates + resos_dates))
if all_dates:
await populate_daily_metrics(db, all_dates)
logger.info("Aggregation completed successfully")
except Exception as e:
logger.error(f"Aggregation failed: {e}")
raise
finally:
db.close()
async def aggregate_newbook_dates(db, dates: List[date]):
"""
Aggregate newbook_bookings into daily_occupancy for specified dates.
Room availability is sourced from newbook_occupancy_report table (preferred)
which provides accurate available rooms accounting for maintenance/offline rooms.
Falls back to system config total_rooms if no occupancy report data exists.
Revenue metrics:
- room_revenue, adr, revpar = NET values (after VAT)
- agr = Actual Guest Rate (gross rate guest paid, from calculated_amount)
"""
logger.info(f"Aggregating {len(dates)} Newbook dates")
# Get fallback total_rooms from system config (used when no occupancy report data)
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'total_rooms'")
)
row = result.fetchone()
fallback_total_rooms = int(row.config_value) if row and row.config_value else 80
# Get accommodation VAT rate from config (default 20%)
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
)
row = result.fetchone()
accommodation_vat = float(row.config_value) if row and row.config_value else 0.20
# Statuses that count as "occupied" (case-insensitive check in query)
# Includes: Confirmed, Unconfirmed, Arrived, Departed, In-House, etc.
# Excludes: Cancelled, No Show, Quote, Waitlist
excluded_statuses = "('cancelled', 'no show', 'no_show', 'quote', 'waitlist')"
# Overflow room category (category_id=5) is used for chargeable no-shows/cancellations
# and should be excluded from room night counts
overflow_category_id = '5'
for d in dates:
# Get room availability from daily_occupancy (pre-calculated by occupancy report sync)
# These values account for maintenance/offline rooms
# Revenue comes from booking data, not occupancy report
result = db.execute(
text("""
SELECT
total_rooms, available_rooms, maintenance_rooms,
newbook_occupied, newbook_occupancy_pct
FROM daily_occupancy
WHERE date = :date
"""),
{"date": d}
)
existing = result.fetchone()
# Use existing availability values if present, else fallback to config
if existing and existing.available_rooms and existing.available_rooms > 0:
total_rooms = existing.total_rooms
available_rooms = existing.available_rooms
maintenance_rooms = existing.maintenance_rooms or 0
newbook_occupied = existing.newbook_occupied
newbook_occupancy_pct = float(existing.newbook_occupancy_pct or 0)
else:
# No occupancy data yet - fall back to config
total_rooms = fallback_total_rooms
available_rooms = fallback_total_rooms # Assume all rooms available
maintenance_rooms = 0
newbook_occupied = None
newbook_occupancy_pct = None
# Calculate occupancy stats for this date from booking data
# A booking is "in house" if: arrival_date <= date < departure_date
# AND status is not cancelled/no-show/quote/waitlist
# EXCLUDES overflow room category (used for chargeable no-shows)
result = db.execute(
text(f"""
SELECT
COUNT(*) as occupied_rooms,
COALESCE(SUM(total_guests), 0) as total_guests,
COALESCE(SUM(adults), 0) as total_adults,
COALESCE(SUM(children), 0) as total_children,
COALESCE(SUM(infants), 0) as total_infants
FROM newbook_bookings
WHERE arrival_date <= :date
AND departure_date > :date
AND LOWER(status) NOT IN {excluded_statuses}
AND (category_id IS NULL OR category_id != :overflow_cat)
"""),
{"date": d, "overflow_cat": overflow_category_id}
)
stats = result.fetchone()
# Count arrivals for this date (active bookings only, excluding overflow)
result = db.execute(
text(f"""
SELECT COUNT(*) as arrival_count
FROM newbook_bookings
WHERE arrival_date = :date
AND LOWER(status) NOT IN {excluded_statuses}
AND (category_id IS NULL OR category_id != :overflow_cat)
"""),
{"date": d, "overflow_cat": overflow_category_id}
)
arrivals = result.fetchone()
# Calculate room revenue, breakfast/dinner allocations from booking_nights
# charge_amount = room rate (net of inventory items, but includes VAT)
# calculated_amount = gross rate guest paid (for AGR)
# GL code matching is done during sync for meal allocations
# EXCLUDES overflow category (chargeable no-shows are not actual room stays)
result = db.execute(
text(f"""
SELECT
COALESCE(SUM(bn.charge_amount), 0) as charge_amount_total,
COALESCE(SUM(bn.calculated_amount), 0) as calculated_amount_total,
COALESCE(SUM(CASE WHEN bn.breakfast_gross > 0 THEN 1 ELSE 0 END), 0) as breakfast_qty,
COALESCE(SUM(bn.breakfast_net), 0) as breakfast_value,
COALESCE(SUM(CASE WHEN bn.dinner_gross > 0 THEN 1 ELSE 0 END), 0) as dinner_qty,
COALESCE(SUM(bn.dinner_net), 0) as dinner_value
FROM newbook_booking_nights bn
JOIN newbook_bookings b ON bn.booking_id = b.id
WHERE bn.stay_date = :date
AND LOWER(b.status) NOT IN {excluded_statuses}
AND (b.category_id IS NULL OR b.category_id != :overflow_cat)
"""),
{"date": d, "overflow_cat": overflow_category_id}
)
revenue_and_meals = result.fetchone()
# Revenue breakdown by room category (for revenue_by_room_type JSON)
# EXCLUDES overflow category from breakdown
result = db.execute(
text(f"""
SELECT
COALESCE(b.category_id, 'unknown') as category_id,
COUNT(DISTINCT b.id) as rooms,
COALESCE(SUM(bn.charge_amount), 0) as charge_amount,
COALESCE(SUM(bn.calculated_amount), 0) as calculated_amount
FROM newbook_booking_nights bn
JOIN newbook_bookings b ON bn.booking_id = b.id
WHERE bn.stay_date = :date
AND LOWER(b.status) NOT IN {excluded_statuses}
AND (b.category_id IS NULL OR b.category_id != :overflow_cat)
GROUP BY b.category_id
"""),
{"date": d, "overflow_cat": overflow_category_id}
)
revenue_by_category_rows = result.fetchall()
# Booking movement stats - count by status category
result = db.execute(
text("""
SELECT
COUNT(*) FILTER (WHERE LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')) as total_bookings,
COUNT(*) FILTER (WHERE LOWER(status) IN ('cancelled')) as cancelled_bookings,
COUNT(*) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')) as no_show_bookings
FROM newbook_bookings
WHERE arrival_date <= :date
AND departure_date > :date
"""),
{"date": d}
)
movement = result.fetchone()
# Breakdown by room category (keyed by category_id for stability)
# EXCLUDES overflow category
result = db.execute(
text(f"""
SELECT
COALESCE(category_id, 'unknown') as category_id,
COUNT(*) as rooms,
COALESCE(SUM(total_guests), 0) as guests,
COALESCE(SUM(adults), 0) as adults,
COALESCE(SUM(children), 0) as children,
COALESCE(SUM(infants), 0) as infants
FROM newbook_bookings
WHERE arrival_date <= :date
AND departure_date > :date
AND LOWER(status) NOT IN {excluded_statuses}
AND (category_id IS NULL OR category_id != :overflow_cat)
GROUP BY category_id
"""),
{"date": d, "overflow_cat": overflow_category_id}
)
room_type_rows = result.fetchall()
# Keyed by category_id - use room_categories table to get names in UI
by_room_type = {}
for row in room_type_rows:
by_room_type[row.category_id] = {
"rooms": row.rooms,
"guests": row.guests,
"adults": row.adults,
"children": row.children,
"infants": row.infants
}
occupied_rooms = stats.occupied_rooms or 0
# Use available_rooms (accounts for maintenance) for accurate occupancy %
occupancy_pct = (occupied_rooms / available_rooms * 100) if available_rooms > 0 else 0
# Calculate revenue metrics
# charge_amount is the room rate (includes VAT), convert to NET
charge_amount_total = float(revenue_and_meals.charge_amount_total or 0)
room_revenue = charge_amount_total / (1 + accommodation_vat) # NET room revenue
# ADR and RevPAR are NET values
# ADR uses occupied rooms, RevPAR uses available rooms
adr = (room_revenue / occupied_rooms) if occupied_rooms > 0 else 0
revpar = (room_revenue / available_rooms) if available_rooms > 0 else 0
# AGR (Actual Guest Rate) = gross rate guest paid (from calculated_amount)
calculated_amount_total = float(revenue_and_meals.calculated_amount_total or 0)
agr = (calculated_amount_total / occupied_rooms) if occupied_rooms > 0 else 0
# Build revenue_by_room_type JSON with net revenue, ADR, AGR per category
revenue_by_room_type = {}
for row in revenue_by_category_rows:
cat_charge = float(row.charge_amount or 0)
cat_calculated = float(row.calculated_amount or 0)
cat_rooms = row.rooms or 0
cat_revenue_net = cat_charge / (1 + accommodation_vat)
revenue_by_room_type[row.category_id] = {
"rooms": cat_rooms,
"revenue_net": round(cat_revenue_net, 2),
"adr_net": round(cat_revenue_net / cat_rooms, 2) if cat_rooms > 0 else 0,
"agr_total": round(cat_calculated, 2),
"agr_avg": round(cat_calculated / cat_rooms, 2) if cat_rooms > 0 else 0
}
# Upsert into daily_occupancy
# Revenue comes from booking data (room_revenue, adr, revpar, agr)
db.execute(
text("""
INSERT INTO daily_occupancy (
date, total_rooms, available_rooms, maintenance_rooms, occupied_rooms, occupancy_pct,
newbook_occupied, newbook_occupancy_pct,
total_guests, total_adults, total_children, total_infants,
arrival_count, total_bookings, cancelled_bookings, no_show_bookings,
room_revenue, adr, revpar, agr,
breakfast_allocation_qty, breakfast_allocation_value,
dinner_allocation_qty, dinner_allocation_value,
by_room_type, revenue_by_room_type, fetched_at
) VALUES (
:date, :total_rooms, :available_rooms, :maintenance_rooms, :occupied_rooms, :occupancy_pct,
:newbook_occupied, :newbook_occupancy_pct,
:total_guests, :total_adults, :total_children, :total_infants,
:arrival_count, :total_bookings, :cancelled_bookings, :no_show_bookings,
:room_revenue, :adr, :revpar, :agr,
:breakfast_qty, :breakfast_value,
:dinner_qty, :dinner_value,
:by_room_type, :revenue_by_room_type, NOW()
)
ON CONFLICT (date) DO UPDATE SET
total_rooms = :total_rooms,
available_rooms = :available_rooms,
maintenance_rooms = :maintenance_rooms,
occupied_rooms = :occupied_rooms,
occupancy_pct = :occupancy_pct,
newbook_occupied = :newbook_occupied,
newbook_occupancy_pct = :newbook_occupancy_pct,
total_guests = :total_guests,
total_adults = :total_adults,
total_children = :total_children,
total_infants = :total_infants,
arrival_count = :arrival_count,
total_bookings = :total_bookings,
cancelled_bookings = :cancelled_bookings,
no_show_bookings = :no_show_bookings,
room_revenue = :room_revenue,
adr = :adr,
revpar = :revpar,
agr = :agr,
breakfast_allocation_qty = :breakfast_qty,
breakfast_allocation_value = :breakfast_value,
dinner_allocation_qty = :dinner_qty,
dinner_allocation_value = :dinner_value,
by_room_type = :by_room_type,
revenue_by_room_type = :revenue_by_room_type,
fetched_at = NOW()
"""),
{
"date": d,
"total_rooms": total_rooms,
"available_rooms": available_rooms,
"maintenance_rooms": maintenance_rooms,
"occupied_rooms": occupied_rooms,
"occupancy_pct": round(occupancy_pct, 2),
"newbook_occupied": newbook_occupied,
"newbook_occupancy_pct": round(newbook_occupancy_pct, 2) if newbook_occupancy_pct is not None else None,
"total_guests": stats.total_guests,
"total_adults": stats.total_adults,
"total_children": stats.total_children,
"total_infants": stats.total_infants,
"arrival_count": arrivals.arrival_count or 0,
"total_bookings": movement.total_bookings or 0,
"cancelled_bookings": movement.cancelled_bookings or 0,
"no_show_bookings": movement.no_show_bookings or 0,
"room_revenue": round(room_revenue, 2),
"adr": round(adr, 2),
"revpar": round(revpar, 2),
"agr": round(agr, 2),
"breakfast_qty": revenue_and_meals.breakfast_qty or 0,
"breakfast_value": revenue_and_meals.breakfast_value or 0,
"dinner_qty": revenue_and_meals.dinner_qty or 0,
"dinner_value": revenue_and_meals.dinner_value or 0,
"by_room_type": json.dumps(by_room_type),
"revenue_by_room_type": json.dumps(revenue_by_room_type)
}
)
# Mark queue entries as processed
db.execute(
text("""
UPDATE aggregation_queue
SET aggregated_at = NOW()
WHERE date = :date AND source = 'newbook' AND aggregated_at IS NULL
"""),
{"date": d}
)
db.commit()
logger.info(f"Aggregated {len(dates)} Newbook dates into daily_occupancy")
def load_opening_hours_mappings(db) -> dict:
"""
Load opening hours to period type mappings from resos_opening_hours_mapping table.
Returns dict: {opening_hour_id: period_type}
Where period_type is one of: 'lunch', 'afternoon', 'dinner', 'ignore'
"""
result = db.execute(text("""
SELECT opening_hour_id, period_type
FROM resos_opening_hours_mapping
WHERE period_type != 'ignore'
"""))
mappings = {}
for row in result.fetchall():
mappings[row.opening_hour_id] = row.period_type
return mappings
async def aggregate_resos_dates(db, dates: List[date]):
"""
Aggregate resos_bookings into daily_covers for specified dates.
Uses opening hours mapping to determine service periods (lunch, afternoon, dinner).
Falls back to time-based logic if no mappings configured.
"""
logger.info(f"Aggregating {len(dates)} Resos dates")
# Load opening hours to period type mappings
oh_mappings = load_opening_hours_mappings(db)
use_oh_mapping = len(oh_mappings) > 0
if use_oh_mapping:
logger.info(f"Using {len(oh_mappings)} opening hours mappings for period detection")
else:
logger.info("No opening hours mappings configured, using time-based period detection")
# Status values that count as "active" (case-insensitive check in query)
# Excludes: Cancelled, No Show
excluded_statuses = "('cancelled', 'no show', 'no_show')"
for d in dates:
# Get hotel occupancy data for this date (for dining rate calculation)
occ_result = db.execute(
text("""
SELECT total_guests FROM daily_occupancy WHERE date = :date
"""),
{"date": d}
)
occ_row = occ_result.fetchone()
total_hotel_residents = occ_row.total_guests if occ_row and occ_row.total_guests else None
# Calculate covers by service period
# If we have opening hours mappings, aggregate by mapped period_type
# Otherwise fall back to simple time-based logic
for period in ['lunch', 'afternoon', 'dinner']:
if use_oh_mapping:
# Get the opening_hour_ids that map to this period
period_oh_ids = [oh_id for oh_id, pt in oh_mappings.items() if pt == period]
if not period_oh_ids:
# No mappings for this period, skip
continue
# Build SQL placeholders for opening_hour_ids
oh_placeholders = ", ".join([f":oh_{i}" for i in range(len(period_oh_ids))])
oh_params = {f"oh_{i}": oh_id for i, oh_id in enumerate(period_oh_ids)}
oh_params["date"] = d
period_filter = f"opening_hour_id IN ({oh_placeholders})"
# Get active booking stats
result = db.execute(
text(f"""
SELECT
COUNT(*) as total_bookings,
COALESCE(SUM(covers), 0) as total_covers,
COALESCE(SUM(CASE WHEN is_hotel_guest THEN covers ELSE 0 END), 0) as hotel_guest_covers,
COALESCE(SUM(CASE WHEN NOT is_hotel_guest OR is_hotel_guest IS NULL THEN covers ELSE 0 END), 0) as external_covers,
COALESCE(SUM(CASE WHEN is_dbb THEN covers ELSE 0 END), 0) as dbb_covers,
COALESCE(SUM(CASE WHEN is_package THEN covers ELSE 0 END), 0) as package_covers
FROM resos_bookings
WHERE booking_date = :date
AND {period_filter}
AND LOWER(status) NOT IN {excluded_statuses}
"""),
oh_params
)
stats = result.fetchone()
# Get cancelled/no-show stats separately
result = db.execute(
text(f"""
SELECT
COUNT(*) FILTER (WHERE LOWER(status) = 'cancelled') as cancelled_bookings,
COALESCE(SUM(covers) FILTER (WHERE LOWER(status) = 'cancelled'), 0) as cancelled_covers,
COUNT(*) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')) as no_show_bookings,
COALESCE(SUM(covers) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')), 0) as no_show_covers
FROM resos_bookings
WHERE booking_date = :date
AND {period_filter}
"""),
oh_params
)
movement = result.fetchone()
# Get source breakdown as JSON
result = db.execute(
text(f"""
SELECT
COALESCE(source, 'unknown') as source,
COUNT(*) as bookings,
COALESCE(SUM(covers), 0) as covers
FROM resos_bookings
WHERE booking_date = :date
AND {period_filter}
AND LOWER(status) NOT IN {excluded_statuses}
GROUP BY source
"""),
oh_params
)
source_rows = result.fetchall()
by_source = {row.source: {"bookings": row.bookings, "covers": row.covers} for row in source_rows}
else:
# Fallback: time-based logic (skip afternoon if using fallback)
if period == 'afternoon':
continue
if period == 'lunch':
time_filter = "booking_time < '15:00'"
else: # dinner
time_filter = "booking_time >= '15:00'"
# Get active booking stats
result = db.execute(
text(f"""
SELECT
COUNT(*) as total_bookings,
COALESCE(SUM(covers), 0) as total_covers,
COALESCE(SUM(CASE WHEN is_hotel_guest THEN covers ELSE 0 END), 0) as hotel_guest_covers,
COALESCE(SUM(CASE WHEN NOT is_hotel_guest OR is_hotel_guest IS NULL THEN covers ELSE 0 END), 0) as external_covers,
COALESCE(SUM(CASE WHEN is_dbb THEN covers ELSE 0 END), 0) as dbb_covers,
COALESCE(SUM(CASE WHEN is_package THEN covers ELSE 0 END), 0) as package_covers
FROM resos_bookings
WHERE booking_date = :date
AND {time_filter}
AND LOWER(status) NOT IN {excluded_statuses}
"""),
{"date": d}
)
stats = result.fetchone()
# Get cancelled/no-show stats separately
result = db.execute(
text(f"""
SELECT
COUNT(*) FILTER (WHERE LOWER(status) = 'cancelled') as cancelled_bookings,
COALESCE(SUM(covers) FILTER (WHERE LOWER(status) = 'cancelled'), 0) as cancelled_covers,
COUNT(*) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')) as no_show_bookings,
COALESCE(SUM(covers) FILTER (WHERE LOWER(status) IN ('no show', 'no_show')), 0) as no_show_covers
FROM resos_bookings
WHERE booking_date = :date
AND {time_filter}
"""),
{"date": d}
)
movement = result.fetchone()
# Get source breakdown as JSON
result = db.execute(
text(f"""
SELECT
COALESCE(source, 'unknown') as source,
COUNT(*) as bookings,
COALESCE(SUM(covers), 0) as covers
FROM resos_bookings
WHERE booking_date = :date
AND {time_filter}
AND LOWER(status) NOT IN {excluded_statuses}
GROUP BY source
"""),
{"date": d}
)
source_rows = result.fetchall()
by_source = {row.source: {"bookings": row.bookings, "covers": row.covers} for row in source_rows}
total_bookings = stats.total_bookings or 0
total_covers = stats.total_covers or 0
avg_party_size = (total_covers / total_bookings) if total_bookings > 0 else 0
hotel_guest_covers = stats.hotel_guest_covers or 0
# Calculate hotel guest dining rate (% of hotel residents who dined this period)
# This enables forecasting: forecast occupancy → apply dining rate → predict hotel guest covers
hotel_guest_dining_rate = None
if total_hotel_residents and total_hotel_residents > 0 and hotel_guest_covers > 0:
hotel_guest_dining_rate = round((hotel_guest_covers / total_hotel_residents) * 100, 2)
# Upsert into daily_covers
db.execute(
text("""
INSERT INTO daily_covers (
date, service_period, total_bookings, total_covers, avg_party_size,
hotel_guest_covers, external_covers, dbb_covers, package_covers,
total_hotel_residents, hotel_guest_dining_rate,
cancelled_bookings, cancelled_covers, no_show_bookings, no_show_covers,
by_source, fetched_at
) VALUES (
:date, :service_period, :total_bookings, :total_covers, :avg_party_size,
:hotel_guest_covers, :external_covers, :dbb_covers, :package_covers,
:total_hotel_residents, :hotel_guest_dining_rate,
:cancelled_bookings, :cancelled_covers, :no_show_bookings, :no_show_covers,
:by_source, NOW()
)
ON CONFLICT (date, service_period) DO UPDATE SET
total_bookings = :total_bookings,
total_covers = :total_covers,
avg_party_size = :avg_party_size,
hotel_guest_covers = :hotel_guest_covers,
external_covers = :external_covers,
dbb_covers = :dbb_covers,
package_covers = :package_covers,
total_hotel_residents = :total_hotel_residents,
hotel_guest_dining_rate = :hotel_guest_dining_rate,
cancelled_bookings = :cancelled_bookings,
cancelled_covers = :cancelled_covers,
no_show_bookings = :no_show_bookings,
no_show_covers = :no_show_covers,
by_source = :by_source,
fetched_at = NOW()
"""),
{
"date": d,
"service_period": period,
"total_bookings": total_bookings,
"total_covers": total_covers,
"avg_party_size": round(avg_party_size, 2),
"hotel_guest_covers": hotel_guest_covers,
"external_covers": stats.external_covers or 0,
"dbb_covers": stats.dbb_covers or 0,
"package_covers": stats.package_covers or 0,
"total_hotel_residents": total_hotel_residents,
"hotel_guest_dining_rate": hotel_guest_dining_rate,
"cancelled_bookings": movement.cancelled_bookings or 0,
"cancelled_covers": movement.cancelled_covers or 0,
"no_show_bookings": movement.no_show_bookings or 0,
"no_show_covers": movement.no_show_covers or 0,
"by_source": json.dumps(by_source)
}
)
# Mark queue entries as processed
db.execute(
text("""
UPDATE aggregation_queue
SET aggregated_at = NOW()
WHERE date = :date AND source = 'resos' AND aggregated_at IS NULL
"""),
{"date": d}
)
db.commit()
logger.info(f"Aggregated {len(dates)} Resos dates into daily_covers")
async def populate_daily_metrics(db, dates: List[date]):
"""
Populate daily_metrics table from daily_occupancy and daily_covers.
This table is the source for forecasting models.
"""
logger.info(f"Populating daily_metrics for {len(dates)} dates")
for d in dates:
# Get daily_occupancy data
result = db.execute(
text("""
SELECT
occupied_rooms, total_guests, total_adults, total_children,
arrival_count, occupancy_pct, adr, revpar,
breakfast_allocation_qty, dinner_allocation_qty,
room_revenue, available_rooms
FROM daily_occupancy
WHERE date = :date
"""),
{"date": d}
)
occupancy = result.fetchone()
# Get daily_covers data (lunch and dinner)
result = db.execute(
text("""
SELECT
service_period, total_bookings, total_covers, avg_party_size
FROM daily_covers
WHERE date = :date
"""),
{"date": d}
)
covers_rows = result.fetchall()
# Build covers data by period
covers_data = {}
for row in covers_rows:
covers_data[row.service_period] = {
"bookings": row.total_bookings,
"covers": row.total_covers,
"party_size": float(row.avg_party_size or 0)
}
# Define metrics to populate
metrics_to_insert = []
if occupancy:
# Hotel metrics
metrics_to_insert.extend([
("hotel_room_nights", occupancy.occupied_rooms, "newbook"),
("hotel_occupancy_pct", float(occupancy.occupancy_pct or 0), "newbook"),
("hotel_guests", occupancy.total_guests, "newbook"),
("hotel_arrivals", occupancy.arrival_count, "newbook"),
("hotel_adr", float(occupancy.adr or 0), "newbook"),
("hotel_revpar", float(occupancy.revpar or 0), "newbook"),
("hotel_breakfast_qty", occupancy.breakfast_allocation_qty, "newbook"),
("hotel_dinner_qty", occupancy.dinner_allocation_qty, "newbook"),
("revenue_rooms", float(occupancy.room_revenue or 0), "newbook"),
])
# Restaurant metrics - lunch
if "lunch" in covers_data:
lunch = covers_data["lunch"]
metrics_to_insert.extend([
("resos_lunch_bookings", lunch["bookings"], "resos"),
("resos_lunch_covers", lunch["covers"], "resos"),
("resos_lunch_party_size", lunch["party_size"], "resos"),
])
# Restaurant metrics - dinner
if "dinner" in covers_data:
dinner = covers_data["dinner"]
metrics_to_insert.extend([
("resos_dinner_bookings", dinner["bookings"], "resos"),
("resos_dinner_covers", dinner["covers"], "resos"),
("resos_dinner_party_size", dinner["party_size"], "resos"),
])
# Insert/update all metrics
for metric_code, actual_value, source in metrics_to_insert:
if actual_value is not None:
db.execute(
text("""
INSERT INTO daily_metrics (date, metric_code, actual_value, source, calculated_at)
VALUES (:date, :metric_code, :actual_value, :source, NOW())
ON CONFLICT (date, metric_code) DO UPDATE SET
actual_value = :actual_value,
source = :source,
calculated_at = NOW()
"""),
{
"date": d,
"metric_code": metric_code,
"actual_value": actual_value,
"source": source
}
)
db.commit()
logger.info(f"Populated daily_metrics for {len(dates)} dates")

456
backend/jobs/ai_insights.py Normal file
View file

@ -0,0 +1,456 @@
"""
AI Daily Insights Generation Job
Gathers Pickup-V2 forecast data, booking pace, competitor rates, and rate parity
information, then sends a compact prompt to Anthropic's Haiku model to generate
a daily briefing for hotel revenue staff.
Schedule: Daily at 7:15 AM (after all forecasts and accuracy calc complete)
Cost: ~$0.05/month at 1 run/day with Haiku
"""
import json
import logging
from datetime import date, timedelta, datetime, timezone
from typing import Dict, List, Any, Optional
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from database import AsyncSessionLocal
logger = logging.getLogger(__name__)
DEFAULT_MODEL = "claude-haiku-4-5-20251001"
DEFAULT_DAILY_TOKEN_BUDGET = 5000
MAX_OUTPUT_TOKENS = 400
async def get_config(db: AsyncSession) -> Dict[str, str]:
"""Get AI insights config from system_config."""
result = await db.execute(
text("""
SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted
FROM system_config
WHERE config_key LIKE 'ai_insights_%'
""")
)
config = {}
for row in result.fetchall():
value = row.config_value
if row.is_encrypted and value:
import base64
try:
value = base64.b64decode(value.encode()).decode()
except Exception:
pass
config[row.config_key] = value
return config
async def check_daily_budget(db: AsyncSession, budget: int) -> tuple[bool, int]:
"""Check if we're within the daily token budget. Returns (within_budget, tokens_used_today)."""
result = await db.execute(
text("""
SELECT COALESCE(SUM(input_tokens + output_tokens), 0) as total
FROM ai_insights
WHERE generated_at >= CURRENT_DATE
""")
)
row = result.fetchone()
used = int(row.total) if row else 0
return used < budget, used
async def gather_occupancy_data(db: AsyncSession, days: int = 14) -> List[Dict]:
"""Gather Pickup-V2 occupancy forecast data for the next N days."""
from services.forecasting.pickup_v2_model import run_pickup_v2_forecast
today = date.today()
end = today + timedelta(days=days - 1)
try:
forecasts = await run_pickup_v2_forecast(
db, 'hotel_occupancy_pct', today, end, include_details=False
)
return forecasts
except Exception as e:
logger.warning(f"Failed to gather occupancy data: {e}")
return []
async def gather_revenue_data(db: AsyncSession, days: int = 14) -> List[Dict]:
"""Gather Pickup-V2 revenue forecast data for the next N days."""
from services.forecasting.pickup_v2_model import run_pickup_v2_forecast
today = date.today()
end = today + timedelta(days=days - 1)
try:
forecasts = await run_pickup_v2_forecast(
db, 'net_accom', today, end, include_details=False
)
return forecasts
except Exception as e:
logger.warning(f"Failed to gather revenue data: {e}")
return []
async def gather_budget_data(db: AsyncSession, days: int = 14) -> Dict[str, float]:
"""Gather budget values for forecast comparison."""
today = date.today()
end = today + timedelta(days=days - 1)
result = await db.execute(
text("""
SELECT date, budget_type, budget_value
FROM daily_budgets
WHERE date BETWEEN :start AND :end
AND budget_type IN ('net_accom', 'occupancy')
"""),
{"start": today, "end": end}
)
budgets = {}
for row in result.fetchall():
key = f"{row.date}_{row.budget_type}"
budgets[key] = float(row.budget_value) if row.budget_value else None
return budgets
async def gather_competitor_data(db: AsyncSession, days: int = 14) -> Dict[str, Any]:
"""Gather competitor rate data from Booking.com scraper."""
today = date.today()
end = today + timedelta(days=days - 1)
# Own hotel rate on Booking.com
own_result = await db.execute(
text("""
SELECT DISTINCT ON (r.rate_date)
r.rate_date,
r.rate_gross as booking_rate,
r.availability_status
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE h.tier = 'own'
AND r.rate_date BETWEEN :start AND :end
ORDER BY r.rate_date, r.scraped_at DESC
"""),
{"start": today, "end": end}
)
own_rates = {str(row.rate_date): {
'rate': float(row.booking_rate) if row.booking_rate else None,
'status': row.availability_status
} for row in own_result.fetchall()}
# Cheapest competitor rate per date
comp_result = await db.execute(
text("""
SELECT DISTINCT ON (r.rate_date)
r.rate_date,
r.rate_gross as comp_rate,
h.name as comp_name
FROM booking_com_rates r
JOIN booking_com_hotels h ON r.hotel_id = h.id
WHERE h.tier IN ('primary', 'secondary')
AND r.rate_date BETWEEN :start AND :end
AND r.availability_status = 'available'
AND r.rate_gross IS NOT NULL
ORDER BY r.rate_date, r.rate_gross ASC
"""),
{"start": today, "end": end}
)
comp_rates = {str(row.rate_date): {
'rate': float(row.comp_rate),
'name': row.comp_name
} for row in comp_result.fetchall()}
# Own rack rate from Newbook
rack_result = await db.execute(
text("""
SELECT DISTINCT ON (rate_date)
rate_date,
rate_gross
FROM newbook_current_rates
WHERE rate_date BETWEEN :start AND :end
ORDER BY rate_date, valid_from DESC
"""),
{"start": today, "end": end}
)
rack_rates = {str(row.rate_date): float(row.rate_gross) if row.rate_gross else None
for row in rack_result.fetchall()}
return {
'own_booking': own_rates,
'competitors': comp_rates,
'rack': rack_rates
}
def build_prompt(
occupancy: List[Dict],
revenue: List[Dict],
budgets: Dict[str, float],
competitor: Dict[str, Any]
) -> tuple[str, str]:
"""Build system and user prompts from gathered data. Returns (system_msg, user_msg)."""
system_msg = (
"You are an AI assistant for a hotel revenue manager. Analyze the data below and provide "
"a concise daily briefing (3-5 bullet points). Focus on: occupancy trends, pace vs prior "
"year, pricing opportunities, rate parity issues, and anything unusual requiring attention. "
"Be specific with numbers and dates. Keep it actionable — no fluff or generic advice."
)
lines = []
# Occupancy section
if occupancy:
lines.append("## Occupancy Forecast - Pickup-V2 (next 14 days)")
lines.append("Date | DoW | OTB | Forecast | PY Final | Pace vs LY | Budget")
for fc in occupancy:
d = fc.get('date', '')
dow = fc.get('day_of_week', '')
otb = fc.get('current_otb')
forecast = fc.get('forecast')
py_final = fc.get('prior_year_final')
pace = fc.get('pace_vs_prior_pct')
budget_key = f"{d}_occupancy"
budget_val = budgets.get(budget_key)
otb_str = f"{otb:.0f}%" if otb is not None else "-"
fc_str = f"{forecast:.0f}%" if forecast is not None else "-"
py_str = f"{py_final:.0f}%" if py_final is not None else "-"
pace_str = f"{pace:+.0f}%" if pace is not None else "-"
bud_str = f"{budget_val:.0f}%" if budget_val is not None else "-"
lines.append(f"{d} | {dow} | {otb_str} | {fc_str} | {py_str} | {pace_str} | {bud_str}")
lines.append("")
# Revenue summary
if revenue:
total_forecast = sum(fc.get('forecast', 0) or 0 for fc in revenue)
total_otb = sum(fc.get('current_otb_rev', 0) or 0 for fc in revenue)
total_py = sum(fc.get('prior_year_final_rev', 0) or 0 for fc in revenue)
opportunity_days = sum(1 for fc in revenue if fc.get('has_pricing_opportunity'))
total_lost = sum(fc.get('lost_potential', 0) or 0 for fc in revenue)
lines.append("## Revenue Signals")
lines.append(f"14-day forecast: ${total_forecast:,.0f} | OTB: ${total_otb:,.0f} | PY: ${total_py:,.0f}")
if opportunity_days > 0:
lines.append(f"Pricing opportunity days: {opportunity_days} | Total lost potential: ${total_lost:,.0f}")
lines.append("")
# Competitor rates section
own_booking = competitor.get('own_booking', {})
comp_rates = competitor.get('competitors', {})
rack_rates = competitor.get('rack', {})
if own_booking or comp_rates:
lines.append("## Competitor Rates (next 14 days)")
lines.append("Date | Own Rack | Own B.com | Cheapest Competitor | Competitor Name")
all_dates = sorted(set(list(own_booking.keys()) + list(comp_rates.keys()) + list(rack_rates.keys())))
for d in all_dates:
rack = rack_rates.get(d)
own = own_booking.get(d, {})
comp = comp_rates.get(d, {})
rack_str = f"${rack:.0f}" if rack else "-"
own_str = f"${own['rate']:.0f}" if own.get('rate') else "-"
comp_str = f"${comp['rate']:.0f}" if comp.get('rate') else "-"
comp_name = comp.get('name', '-')
note = ""
if own.get('rate') and comp.get('rate') and own['rate'] < comp['rate']:
note = " <- cheapest on B.com"
lines.append(f"{d} | {rack_str} | {own_str} | {comp_str} | {comp_name}{note}")
lines.append("")
# Rate parity flags
parity_flags = []
for d in sorted(rack_rates.keys()):
rack = rack_rates.get(d)
own = own_booking.get(d, {})
if rack and own.get('rate') and rack > 0:
delta_pct = ((own['rate'] - rack) / rack) * 100
if abs(delta_pct) > 5:
parity_flags.append(f"{d}: Rack ${rack:.0f} vs B.com ${own['rate']:.0f} ({delta_pct:+.1f}%)")
if parity_flags:
lines.append("## Rate Parity Flags (own rack vs own Booking.com, >5% delta)")
for flag in parity_flags:
lines.append(flag)
lines.append("")
user_msg = "\n".join(lines)
return system_msg, user_msg
async def call_llm(api_key: str, system_msg: str, user_msg: str, model: str) -> Dict[str, Any]:
"""Call Anthropic API and return response with token usage."""
import anthropic
client = anthropic.AsyncAnthropic(api_key=api_key)
try:
response = await client.messages.create(
model=model,
max_tokens=MAX_OUTPUT_TOKENS,
temperature=0.2,
system=system_msg,
messages=[{"role": "user", "content": user_msg}]
)
content = response.content[0].text if response.content else ""
return {
"content": content,
"input_tokens": response.usage.input_tokens,
"output_tokens": response.usage.output_tokens,
"model": model,
}
finally:
await client.close()
async def save_insight(
db: AsyncSession,
content: str,
model: str,
input_tokens: int,
output_tokens: int,
data_snapshot: Dict,
triggered_by: str = "scheduler"
):
"""Save generated insight to database."""
await db.execute(
text("""
INSERT INTO ai_insights
(content, model, input_tokens, output_tokens, data_snapshot, triggered_by)
VALUES (:content, :model, :input_tokens, :output_tokens,
CAST(:data_snapshot AS jsonb), :triggered_by)
"""),
{
"content": content,
"model": model,
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"data_snapshot": json.dumps(data_snapshot),
"triggered_by": triggered_by,
}
)
await db.commit()
async def cleanup_old_insights(db: AsyncSession, keep_days: int = 90):
"""Remove insights older than keep_days."""
cutoff = datetime.now(timezone.utc) - timedelta(days=keep_days)
await db.execute(
text("DELETE FROM ai_insights WHERE generated_at < :cutoff"),
{"cutoff": cutoff}
)
await db.commit()
async def generate_insight(db: AsyncSession, triggered_by: str = "scheduler") -> Dict[str, Any]:
"""
Core insight generation logic. Used by both scheduler and manual trigger.
Returns result dict with success/error status.
"""
config = await get_config(db)
# Check enabled
if config.get('ai_insights_enabled', 'false').lower() not in ('true', '1', 'yes'):
return {"success": False, "error": "AI insights disabled"}
# Check API key
api_key = config.get('ai_insights_api_key')
if not api_key:
return {"success": False, "error": "No API key configured"}
model = config.get('ai_insights_model', DEFAULT_MODEL)
budget = int(config.get('ai_insights_daily_token_budget', str(DEFAULT_DAILY_TOKEN_BUDGET)))
# Check daily budget
within_budget, used_today = await check_daily_budget(db, budget)
if not within_budget:
return {
"success": False,
"error": f"Daily token budget exceeded ({used_today}/{budget} tokens used today)"
}
# Gather data
logger.info("Gathering data for AI insight...")
occupancy = await gather_occupancy_data(db)
revenue = await gather_revenue_data(db)
budgets = await gather_budget_data(db)
competitor = await gather_competitor_data(db)
if not occupancy and not revenue:
return {"success": False, "error": "No forecast data available"}
# Build prompt
system_msg, user_msg = build_prompt(occupancy, revenue, budgets, competitor)
# Store data snapshot for debugging
data_snapshot = {
"occupancy_days": len(occupancy),
"revenue_days": len(revenue),
"competitor_dates": len(competitor.get('own_booking', {})),
"parity_flags": sum(1 for d in competitor.get('rack', {})
if competitor.get('own_booking', {}).get(d, {}).get('rate')
and competitor['rack'].get(d)
and abs((competitor['own_booking'][d]['rate'] - competitor['rack'][d]) / competitor['rack'][d] * 100) > 5),
"prompt_preview": user_msg[:500],
}
# Call LLM
logger.info(f"Calling {model} for AI insight...")
try:
result = await call_llm(api_key, system_msg, user_msg, model)
except Exception as e:
logger.error(f"LLM call failed: {e}")
return {"success": False, "error": f"LLM call failed: {str(e)}"}
# Save
await save_insight(
db,
content=result["content"],
model=result["model"],
input_tokens=result["input_tokens"],
output_tokens=result["output_tokens"],
data_snapshot=data_snapshot,
triggered_by=triggered_by,
)
logger.info(
f"AI insight generated: {result['input_tokens']} input, "
f"{result['output_tokens']} output tokens ({triggered_by})"
)
# Cleanup old insights
try:
await cleanup_old_insights(db)
except Exception as e:
logger.warning(f"Old insight cleanup failed: {e}")
return {
"success": True,
"content": result["content"],
"input_tokens": result["input_tokens"],
"output_tokens": result["output_tokens"],
"model": result["model"],
}
async def run_ai_insights_generation():
"""Scheduled job entry point."""
async with AsyncSessionLocal() as db:
try:
result = await generate_insight(db, triggered_by="scheduler")
if result.get("success"):
logger.info("Scheduled AI insight generation completed")
else:
logger.info(f"Scheduled AI insight skipped: {result.get('error')}")
except Exception as e:
logger.error(f"Scheduled AI insight generation failed: {e}", exc_info=True)

1647
backend/jobs/batch_backtest.py Normal file
View file

File diff suppressed because it is too large Load diff

847
backend/jobs/bookings_aggregation.py Normal file
View file

@ -0,0 +1,847 @@
"""
Bookings aggregation job - aggregates newbook_bookings_data into:
- newbook_bookings_stats: daily aggregated stats with JSONB category breakdowns
- newbook_booking_pace: lead-time snapshots for forecasting pickup patterns
Triggered automatically after bookings sync completes.
"""
import json
import logging
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import List, Set, Dict, Any, Optional
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
# Valid booking statuses for aggregation
VALID_STATUSES = ('Unconfirmed', 'Confirmed', 'Arrived', 'Departed')
# All tracked pace intervals
PACE_INTERVALS = [
# Monthly (months 7-12)
365, 330, 300, 270, 240, 210,
# Weekly (weeks 5-25)
177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
# Daily (days 0-30)
30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
]
def get_config_value(db, key: str) -> Optional[str]:
"""Get a configuration value from system_config table."""
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = :key"),
{"key": key}
)
row = result.fetchone()
return row.config_value if row else None
async def run_bookings_aggregation(triggered_by: str = "manual"):
"""
Aggregate bookings into newbook_bookings_stats.
Flow:
1. Find bookings changed since last_bookings_aggregation_at
2. Calculate affected dates (arrival_date <= date < departure_date)
3. Reaggregate each affected date
4. Update booking pace table
5. Update last_bookings_aggregation_at
"""
logger.info(f"Starting bookings aggregation (triggered_by={triggered_by})")
db = next(iter([SyncSessionLocal()]))
try:
# Get last aggregation timestamp
last_aggregation = get_config_value(db, 'last_bookings_aggregation_at')
if last_aggregation:
try:
last_ts = datetime.fromisoformat(last_aggregation)
except ValueError:
last_ts = datetime.min
else:
last_ts = datetime.min
logger.info(f"Last aggregation: {last_ts}")
# Find bookings changed since last aggregation
result = db.execute(
text("""
SELECT newbook_id, arrival_date, departure_date
FROM newbook_bookings_data
WHERE fetched_at > :last_ts
"""),
{"last_ts": last_ts}
)
changed_bookings = result.fetchall()
if not changed_bookings:
logger.info("No changed bookings to aggregate")
# Still update pace table
await update_booking_pace(db)
db.commit()
return
logger.info(f"Found {len(changed_bookings)} changed bookings")
# Calculate affected dates
affected_dates: Set[date] = set()
for booking in changed_bookings:
if booking.arrival_date and booking.departure_date:
current = booking.arrival_date
# < not <= (departure is checkout day, guest not staying that night)
while current < booking.departure_date:
affected_dates.add(current)
current += timedelta(days=1)
logger.info(f"Reaggregating {len(affected_dates)} affected dates")
# Get accommodation VAT rate
vat_rate_str = get_config_value(db, 'accommodation_vat_rate')
vat_rate = Decimal(vat_rate_str) if vat_rate_str else Decimal('0.20')
# Aggregate each affected date
for target_date in sorted(affected_dates):
await aggregate_date(db, target_date, vat_rate)
# Fill any dates with occupancy data but no bookings (e.g., closed periods)
await fill_occupancy_only_dates(db, vat_rate)
# Update booking pace table
await update_booking_pace(db)
# Update last aggregation timestamp
db.execute(
text("""
INSERT INTO system_config (config_key, config_value, updated_at)
VALUES ('last_bookings_aggregation_at', :now, NOW())
ON CONFLICT (config_key) DO UPDATE SET
config_value = :now,
updated_at = NOW()
"""),
{"now": datetime.now().isoformat()}
)
db.commit()
logger.info(f"Bookings aggregation completed: {len(affected_dates)} dates processed")
except Exception as e:
logger.error(f"Bookings aggregation failed: {e}")
db.rollback()
raise
finally:
db.close()
async def aggregate_date(db, target_date: date, vat_rate: Decimal):
"""
Aggregate all bookings for a specific date into newbook_bookings_stats.
Includes room availability from newbook_occupancy_report_data and
booking stats from newbook_bookings_data.
"""
# Step 1: Get room availability from occupancy report (included categories only)
result = db.execute(
text("""
SELECT
o.category_id,
COALESCE(o.available, 0) as available,
COALESCE(o.maintenance, 0) as maintenance
FROM newbook_occupancy_report_data o
JOIN newbook_room_categories c ON o.category_id = c.site_id
WHERE o.date = :target_date
AND c.is_included = true
"""),
{"target_date": target_date}
)
occupancy_rows = result.fetchall()
# Build availability by category
availability_by_category: Dict[str, Dict[str, Any]] = {}
rooms_count = 0
maintenance_count = 0
for row in occupancy_rows:
cat_id = row.category_id
available = row.available or 0
maintenance = row.maintenance or 0
bookable = available - maintenance
rooms_count += available
maintenance_count += maintenance
availability_by_category[cat_id] = {
"rooms_count": available,
"maintenance_count": maintenance,
"bookable_count": bookable,
"booking_count": 0,
"total_occupancy_pct": None,
"bookable_occupancy_pct": None
}
bookable_count = rooms_count - maintenance_count
# Fallback: If no occupancy data (bookable_count=0), use last known bookable_count
# This prevents division-by-zero issues in forecast models when occupancy report is missing
if bookable_count <= 0:
fallback_result = db.execute(
text("""
SELECT bookable_count
FROM newbook_bookings_stats
WHERE bookable_count > 5 AND date < :target_date
ORDER BY date DESC
LIMIT 1
"""),
{"target_date": target_date}
)
fallback_row = fallback_result.fetchone()
if fallback_row and fallback_row.bookable_count:
bookable_count = fallback_row.bookable_count
rooms_count = bookable_count # Assume same for rooms_count
logger.info(f"Using fallback bookable_count={bookable_count} for {target_date}")
# Step 2: Get booking stats (bookings staying this night)
# A booking is "in house" if: arrival_date <= date < departure_date
# Only counts bookings for categories marked as is_included=true in settings
result = db.execute(
text("""
SELECT
b.newbook_id,
b.category_id,
COALESCE(b.adults, 0) + COALESCE(b.children, 0) + COALESCE(b.infants, 0) as guests,
COALESCE(b.adults, 0) as adults,
COALESCE(b.children, 0) as children,
COALESCE(b.infants, 0) as infants,
b.raw_json
FROM newbook_bookings_data b
JOIN newbook_room_categories c ON b.category_id = c.site_id
WHERE b.arrival_date <= :target_date
AND b.departure_date > :target_date
AND b.status IN :valid_statuses
AND c.is_included = true
"""),
{"target_date": target_date, "valid_statuses": VALID_STATUSES}
)
bookings = result.fetchall()
# Aggregate bookings
booking_count = 0
guests_count = 0
adults_count = 0
children_count = 0
infants_count = 0
guest_rate_total = Decimal('0')
net_booking_rev_total = Decimal('0')
occupancy_by_category: Dict[str, Dict[str, Any]] = {}
revenue_by_category: Dict[str, Dict[str, Any]] = {}
rate_stats_by_category: Dict[str, Dict[str, Any]] = {} # Pickup-V2: min/max/adr per category
for booking in bookings:
booking_count += 1
guests_count += booking.guests or 0
adults_count += booking.adults or 0
children_count += booking.children or 0
infants_count += booking.infants or 0
cat_id = booking.category_id or 'unknown'
# Initialize category dicts if needed
if cat_id not in occupancy_by_category:
occupancy_by_category[cat_id] = {
"booking_count": 0,
"guests": 0,
"adults": 0,
"children": 0,
"infants": 0
}
if cat_id not in revenue_by_category:
revenue_by_category[cat_id] = {
"guest_rate_total": Decimal('0'),
"net_booking_rev_total": Decimal('0')
}
if cat_id not in rate_stats_by_category:
rate_stats_by_category[cat_id] = {
"rates": [], # Collect all net rates for min/max/adr calculation
"rooms": 0
}
# Update occupancy by category
occupancy_by_category[cat_id]["booking_count"] += 1
occupancy_by_category[cat_id]["guests"] += booking.guests or 0
occupancy_by_category[cat_id]["adults"] += booking.adults or 0
occupancy_by_category[cat_id]["children"] += booking.children or 0
occupancy_by_category[cat_id]["infants"] += booking.infants or 0
# Update availability by category booking count
if cat_id in availability_by_category:
availability_by_category[cat_id]["booking_count"] += 1
# Get revenue from tariffs_quoted for this date
calculated_amount, net_amount = get_rate_for_date(
booking.raw_json, target_date, vat_rate
)
guest_rate_total += calculated_amount
net_booking_rev_total += net_amount
revenue_by_category[cat_id]["guest_rate_total"] += calculated_amount
revenue_by_category[cat_id]["net_booking_rev_total"] += net_amount
# Pickup-V2: Collect net rate for rate stats (only if rate > 0)
if net_amount > 0:
rate_stats_by_category[cat_id]["rates"].append(float(net_amount))
rate_stats_by_category[cat_id]["rooms"] += 1
# Calculate occupancy percentages
total_occupancy_pct = None
bookable_occupancy_pct = None
if rooms_count > 0:
total_occupancy_pct = round(float(booking_count) / rooms_count * 100, 2)
if bookable_count > 0:
bookable_occupancy_pct = round(float(booking_count) / bookable_count * 100, 2)
# Calculate per-category occupancy percentages
for cat_id, avail in availability_by_category.items():
cat_bookings = avail["booking_count"]
cat_rooms = avail["rooms_count"]
cat_bookable = avail["bookable_count"]
if cat_rooms > 0:
avail["total_occupancy_pct"] = round(float(cat_bookings) / cat_rooms * 100, 2)
if cat_bookable > 0:
avail["bookable_occupancy_pct"] = round(float(cat_bookings) / cat_bookable * 100, 2)
# Convert Decimal to float for JSON serialization
def decimal_to_float(d: Dict) -> Dict:
return {
k: (float(v) if isinstance(v, Decimal) else v)
for k, v in d.items()
}
# Pickup-V2: Calculate min/max/adr from collected rates
rate_stats_final: Dict[str, Dict[str, Any]] = {}
for cat_id, stats in rate_stats_by_category.items():
rates = stats["rates"]
if rates:
rate_stats_final[cat_id] = {
"min_net": round(min(rates), 2),
"max_net": round(max(rates), 2),
"adr_net": round(sum(rates) / len(rates), 2),
"rooms": stats["rooms"]
}
occupancy_json = json.dumps({
k: decimal_to_float(v) for k, v in occupancy_by_category.items()
})
revenue_json = json.dumps({
k: decimal_to_float(v) for k, v in revenue_by_category.items()
})
availability_json = json.dumps(availability_by_category)
rate_stats_json = json.dumps(rate_stats_final)
# Upsert into newbook_bookings_stats
db.execute(
text("""
INSERT INTO newbook_bookings_stats (
date, rooms_count, maintenance_count, bookable_count,
booking_count, guests_count, adults_count, children_count, infants_count,
total_occupancy_pct, bookable_occupancy_pct,
guest_rate_total, net_booking_rev_total,
occupancy_by_category, revenue_by_category, availability_by_category,
rate_stats_by_category,
aggregated_at
) VALUES (
:date, :rooms_count, :maintenance_count, :bookable_count,
:booking_count, :guests_count, :adults_count, :children_count, :infants_count,
:total_occupancy_pct, :bookable_occupancy_pct,
:guest_rate_total, :net_booking_rev_total,
:occupancy_by_category, :revenue_by_category, :availability_by_category,
:rate_stats_by_category,
NOW()
)
ON CONFLICT (date) DO UPDATE SET
rooms_count = :rooms_count,
maintenance_count = :maintenance_count,
bookable_count = :bookable_count,
booking_count = :booking_count,
guests_count = :guests_count,
adults_count = :adults_count,
children_count = :children_count,
infants_count = :infants_count,
total_occupancy_pct = :total_occupancy_pct,
bookable_occupancy_pct = :bookable_occupancy_pct,
guest_rate_total = :guest_rate_total,
net_booking_rev_total = :net_booking_rev_total,
occupancy_by_category = :occupancy_by_category,
revenue_by_category = :revenue_by_category,
availability_by_category = :availability_by_category,
rate_stats_by_category = :rate_stats_by_category,
aggregated_at = NOW()
"""),
{
"date": target_date,
"rooms_count": rooms_count,
"maintenance_count": maintenance_count,
"bookable_count": bookable_count,
"booking_count": booking_count,
"guests_count": guests_count,
"adults_count": adults_count,
"children_count": children_count,
"infants_count": infants_count,
"total_occupancy_pct": total_occupancy_pct,
"bookable_occupancy_pct": bookable_occupancy_pct,
"guest_rate_total": float(guest_rate_total),
"net_booking_rev_total": float(net_booking_rev_total),
"occupancy_by_category": occupancy_json,
"revenue_by_category": revenue_json,
"availability_by_category": availability_json,
"rate_stats_by_category": rate_stats_json
}
)
def get_rate_for_date(raw_json: dict, target_date: date, vat_rate: Decimal) -> tuple:
"""
Extract rate from tariffs_quoted for specific stay_date.
Returns tuple of (calculated_amount, net_amount).
calculated_amount = gross rate guest paid (for AGR)
net_amount = amount after VAT deduction
"""
if not raw_json:
return Decimal('0'), Decimal('0')
tariffs = raw_json.get("tariffs_quoted", [])
target_str = target_date.strftime("%Y-%m-%d")
for tariff in tariffs:
if tariff.get("stay_date") == target_str:
calculated_amount = Decimal(str(tariff.get("calculated_amount", 0) or 0))
charge_amount = Decimal(str(tariff.get("charge_amount", 0) or 0))
# Try to get net from taxes array if available
taxes = tariff.get("taxes", [])
if taxes and charge_amount > 0:
tax_amount = sum(Decimal(str(t.get("tax_amount", 0) or 0)) for t in taxes)
net_amount = charge_amount - tax_amount
else:
# Fallback: calculate net using VAT rate
net_amount = charge_amount / (1 + vat_rate)
return calculated_amount, net_amount
return Decimal('0'), Decimal('0')
async def update_booking_pace(db):
"""
Update booking pace table with current snapshots.
For each tracked interval, snapshot the current OCCUPANCY count for that stay_date.
Occupancy = arrivals + stayovers (guests already checked in from earlier dates).
This counts bookings where: arrival_date <= stay_date < departure_date
Also ensures all dates in the forecast window have rows (prevents gaps when job misses a day).
"""
logger.info("Updating booking pace snapshots (occupancy-based)")
today = date.today()
updates = 0
# Step 1: Update tracked interval columns
for interval in PACE_INTERVALS:
stay_date = today + timedelta(days=interval)
# Count OCCUPANCY for this stay_date (arrivals + stayovers)
# A booking occupies a date if: arrival_date <= stay_date < departure_date
result = db.execute(
text("""
SELECT COUNT(*) as count
FROM newbook_bookings_data b
JOIN newbook_room_categories c ON b.category_id = c.site_id
WHERE b.arrival_date <= :stay_date
AND b.departure_date > :stay_date
AND b.status IN :valid_statuses
AND c.is_included = true
"""),
{"stay_date": stay_date, "valid_statuses": VALID_STATUSES}
)
row = result.fetchone()
booking_count = row.count if row else 0
# Upsert to pace table (column still named arrival_date for backwards compat)
column_name = f"d{interval}"
# Build dynamic SQL for upsert
db.execute(
text(f"""
INSERT INTO newbook_booking_pace (arrival_date, {column_name}, updated_at)
VALUES (:stay_date, :count, NOW())
ON CONFLICT (arrival_date) DO UPDATE
SET {column_name} = :count, updated_at = NOW()
"""),
{"stay_date": stay_date, "count": booking_count}
)
updates += 1
# Step 2: Update gap dates (31-36, 38-43, etc.) with their bracketed column
# These dates fall between tracked intervals and need their nearest column updated
gap_updates = 0
for days_out in range(31, 90): # Cover the gap range where intervals are weekly
if days_out in PACE_INTERVALS:
continue # Already handled in step 1
stay_date = today + timedelta(days=days_out)
# Find the bracketed column (round up to next interval)
bracket_col = None
for interval in sorted(PACE_INTERVALS):
if interval >= days_out:
bracket_col = f"d{interval}"
break
if not bracket_col:
continue
# Count OCCUPANCY (arrivals + stayovers)
result = db.execute(
text("""
SELECT COUNT(*) as count
FROM newbook_bookings_data b
JOIN newbook_room_categories c ON b.category_id = c.site_id
WHERE b.arrival_date <= :stay_date
AND b.departure_date > :stay_date
AND b.status IN :valid_statuses
AND c.is_included = true
"""),
{"stay_date": stay_date, "valid_statuses": VALID_STATUSES}
)
row = result.fetchone()
booking_count = row.count if row else 0
# Upsert with the bracketed column
db.execute(
text(f"""
INSERT INTO newbook_booking_pace (arrival_date, {bracket_col}, updated_at)
VALUES (:stay_date, :count, NOW())
ON CONFLICT (arrival_date) DO UPDATE
SET {bracket_col} = :count, updated_at = NOW()
"""),
{"stay_date": stay_date, "count": booking_count}
)
gap_updates += 1
logger.info(f"Updated {updates} pace snapshots + {gap_updates} gap dates (occupancy-based)")
async def fill_occupancy_only_dates(db, vat_rate: Decimal = None):
"""
Create stats rows for dates that have occupancy data but no bookings.
This ensures dates like closed periods (all rooms in maintenance) get proper
stats rows with bookable_count=0, so forecasts can cap correctly.
"""
if vat_rate is None:
vat_rate_str = get_config_value(db, 'accommodation_vat_rate')
vat_rate = Decimal(vat_rate_str) if vat_rate_str else Decimal('0.20')
# Find dates with occupancy data but no stats row
result = db.execute(
text("""
SELECT DISTINCT o.date
FROM newbook_occupancy_report_data o
JOIN newbook_room_categories c ON o.category_id = c.site_id
WHERE c.is_included = true
AND NOT EXISTS (
SELECT 1 FROM newbook_bookings_stats s WHERE s.date = o.date
)
ORDER BY o.date
""")
)
missing_dates = [row.date for row in result.fetchall()]
if not missing_dates:
logger.info("No occupancy-only dates to fill")
return 0
logger.info(f"Filling {len(missing_dates)} occupancy-only dates (no bookings)")
for target_date in missing_dates:
await aggregate_date(db, target_date, vat_rate)
logger.info(f"Filled {len(missing_dates)} occupancy-only dates")
return len(missing_dates)
async def backfill_aggregation(db=None):
"""
Backfill historical data into newbook_bookings_stats and newbook_booking_pace.
- Stats: Aggregates all dates that have bookings staying
- Pace: Reconstructs historical snapshots using booking_placed timestamps
"""
import sys
print("[BACKFILL] Starting backfill aggregation...", flush=True)
sys.stdout.flush()
close_db = False
if db is None:
db = next(iter([SyncSessionLocal()]))
close_db = True
try:
# Get VAT rate
vat_rate_str = get_config_value(db, 'accommodation_vat_rate')
vat_rate = Decimal(vat_rate_str) if vat_rate_str else Decimal('0.20')
# Step 1: Get all unique stay dates from bookings
print("[BACKFILL] Finding all stay dates...", flush=True)
result = db.execute(
text("""
SELECT DISTINCT d::date as stay_date
FROM newbook_bookings_data b,
generate_series(b.arrival_date, b.departure_date - interval '1 day', '1 day') d
WHERE b.status IN :valid_statuses
ORDER BY stay_date
"""),
{"valid_statuses": VALID_STATUSES}
)
stay_dates = [row.stay_date for row in result.fetchall()]
print(f"[BACKFILL] Found {len(stay_dates)} stay dates to aggregate", flush=True)
# Step 2: Aggregate each stay date into stats
for i, target_date in enumerate(stay_dates):
if i % 100 == 0:
print(f"[BACKFILL] Aggregating stats: {i}/{len(stay_dates)} dates...", flush=True)
db.commit() # Commit periodically
await aggregate_date(db, target_date, vat_rate)
db.commit()
print(f"[BACKFILL] Stats aggregation complete: {len(stay_dates)} dates", flush=True)
# Step 2b: Fill in dates with occupancy data but no bookings (e.g., closed periods)
print("[BACKFILL] Filling occupancy-only dates (no bookings)...", flush=True)
filled_count = await fill_occupancy_only_dates(db, vat_rate)
db.commit()
print(f"[BACKFILL] Filled {filled_count} occupancy-only dates", flush=True)
# Step 3: Get ALL dates from stats for pace backfill
# This includes dates with 0 bookings (closed periods, future dates)
# Critical: Without pace entries, models may predict 100% occupancy
print("[BACKFILL] Finding all stats dates for pace...", flush=True)
result = db.execute(
text("""
SELECT date as stay_date
FROM newbook_bookings_stats
ORDER BY date
""")
)
stay_dates_for_pace = [row.stay_date for row in result.fetchall()]
print(f"[BACKFILL] Found {len(stay_dates_for_pace)} stats dates for pace backfill", flush=True)
# Step 4: Backfill pace for each stay date (occupancy-based)
today = date.today()
for i, stay_date in enumerate(stay_dates_for_pace):
if i % 100 == 0:
print(f"[BACKFILL] Backfilling pace: {i}/{len(stay_dates_for_pace)} dates...", flush=True)
db.commit()
await backfill_pace_for_date(db, stay_date, today)
db.commit()
print(f"[BACKFILL] Pace backfill complete: {len(stay_dates_for_pace)} dates (occupancy-based)", flush=True)
# Update last aggregation timestamp
db.execute(
text("""
INSERT INTO system_config (config_key, config_value, updated_at)
VALUES ('last_bookings_aggregation_at', :now, NOW())
ON CONFLICT (config_key) DO UPDATE SET
config_value = :now,
updated_at = NOW()
"""),
{"now": datetime.now().isoformat()}
)
db.commit()
print("[BACKFILL] Backfill complete!", flush=True)
logger.info("Backfill aggregation completed successfully")
except Exception as e:
print(f"[BACKFILL] FAILED: {e}", flush=True)
logger.error(f"Backfill aggregation failed: {e}")
db.rollback()
raise
finally:
if close_db:
db.close()
async def backfill_pace_for_date(db, stay_date: date, today: date):
"""
Backfill pace snapshots for a single stay date using booking_placed timestamps.
Tracks OCCUPANCY (arrivals + stayovers), not just arrivals.
For historical stays: Reconstruct what occupancy would have been at each lead time
For future stays: Use current count for today's lead time
"""
# For each interval, calculate what the occupancy count was at that point
# Using booking_placed to determine when each booking was created
pace_values = {}
for interval in PACE_INTERVALS:
# The snapshot date is when we would have taken this measurement
snapshot_date = stay_date - timedelta(days=interval)
if snapshot_date > today:
# This snapshot hasn't happened yet - skip
continue
if snapshot_date < date(2020, 1, 1):
# Don't go too far back - skip ancient dates
continue
# Count OCCUPANCY that existed at the snapshot date
# A booking contributes to occupancy if:
# - arrival_date <= stay_date < departure_date (booking spans this night)
# - booking_placed <= snapshot_date (booking existed at measurement time)
# Only counts categories with is_included = true
result = db.execute(
text("""
SELECT COUNT(*) as count
FROM newbook_bookings_data b
JOIN newbook_room_categories c ON b.category_id = c.site_id
WHERE b.arrival_date <= :stay_date
AND b.departure_date > :stay_date
AND b.status IN :valid_statuses
AND c.is_included = true
AND b.booking_placed IS NOT NULL
AND b.booking_placed::date <= :snapshot_date
"""),
{
"stay_date": stay_date,
"valid_statuses": VALID_STATUSES,
"snapshot_date": snapshot_date
}
)
row = result.fetchone()
pace_values[f"d{interval}"] = row.count if row else 0
if not pace_values:
return
# Build dynamic upsert for all columns we have values for
columns = list(pace_values.keys())
set_clauses = ", ".join([f"{col} = :{col}" for col in columns])
insert_cols = ", ".join(columns)
insert_vals = ", ".join([f":{col}" for col in columns])
db.execute(
text(f"""
INSERT INTO newbook_booking_pace (arrival_date, {insert_cols}, updated_at)
VALUES (:stay_date, {insert_vals}, NOW())
ON CONFLICT (arrival_date) DO UPDATE SET
{set_clauses}, updated_at = NOW()
"""),
{"stay_date": stay_date, **pace_values}
)
async def fill_missing_pace_entries(db=None):
"""
Fill pace entries for all stats dates that don't have pace rows.
This fixes gaps where dates exist in stats (with 0 or more bookings)
but have no pace data, causing models to predict incorrectly.
"""
import sys
print("[PACE-FILL] Finding dates missing pace entries...", flush=True)
close_db = False
if db is None:
db = next(iter([SyncSessionLocal()]))
close_db = True
try:
# Find dates in stats but not in pace
result = db.execute(
text("""
SELECT s.date as stay_date
FROM newbook_bookings_stats s
LEFT JOIN newbook_booking_pace p ON s.date = p.arrival_date
WHERE p.arrival_date IS NULL
ORDER BY s.date
""")
)
missing_dates = [row.stay_date for row in result.fetchall()]
if not missing_dates:
print("[PACE-FILL] No missing pace entries found", flush=True)
return 0
print(f"[PACE-FILL] Found {len(missing_dates)} dates missing pace entries", flush=True)
today = date.today()
for i, stay_date in enumerate(missing_dates):
if i % 100 == 0:
print(f"[PACE-FILL] Processing: {i}/{len(missing_dates)} dates...", flush=True)
db.commit()
await backfill_pace_for_date(db, stay_date, today)
db.commit()
print(f"[PACE-FILL] Filled {len(missing_dates)} missing pace entries", flush=True)
return len(missing_dates)
except Exception as e:
print(f"[PACE-FILL] FAILED: {e}", flush=True)
db.rollback()
raise
finally:
if close_db:
db.close()
def get_pace_interval(days_out: int) -> str:
"""
Get the pace column to use for a given lead time.
Uses round-up logic (next higher interval for conservative estimates).
Examples:
- 25 days out → d25 (exact daily match)
- 35 days out → d37 (rounds up to next weekly)
- 200 days out → d210 (rounds up to next monthly)
"""
# Monthly thresholds (7-12 months)
if days_out >= 365:
return "d365"
if days_out >= 330:
return "d365"
if days_out >= 300:
return "d330"
if days_out >= 270:
return "d300"
if days_out >= 240:
return "d270"
if days_out >= 210:
return "d240"
# Weekly thresholds (5-25 weeks)
weekly = [177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37]
for i, threshold in enumerate(weekly):
if days_out >= threshold:
return f"d{weekly[i - 1]}" if i > 0 else "d210"
# Daily (0-30 days) - exact match available
if days_out > 30:
return "d37" # Round up to first weekly
return f"d{days_out}"

1252
backend/jobs/data_sync.py Normal file
View file

File diff suppressed because it is too large Load diff

359
backend/jobs/fetch_current_rates.py Normal file
View file

@ -0,0 +1,359 @@
"""
Fetch Current Rates Job
Fetches current rack rates from Newbook API and populates newbook_current_rates table.
These rates are used by pickup-v2 model for upper bound calculations in confidence shading.
Uses a snapshot model - only inserts new rows when rates change, otherwise updates last_verified_at.
This allows tracking rate history over time.
Schedule: Daily at 5:20 AM (before pace snapshot runs)
Processing: Day-by-day with progressive DB commits. Each date is fully processed
(single-night fetch + inline multi-night verification) and saved before moving to the next.
If the job fails partway, all previously processed dates are preserved.
"""
import json
import logging
import re
from datetime import date, timedelta
from decimal import Decimal
from typing import Dict, Any, Optional, Set
import asyncio
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
COMMIT_BATCH_SIZE = 10 # Commit to DB every N days
def rates_changed(old_rate: Optional[Dict], new_rate: Dict) -> bool:
"""
Compare old and new rates to determine if they've changed.
Compares gross rate, net rate, and tariff availability status.
Returns True if rates have changed, False if they're the same.
"""
if old_rate is None:
return True # No existing rate, need to insert
# Compare gross and net rates
old_gross = float(old_rate.get('rate_gross') or 0)
new_gross = float(new_rate.get('gross_rate') or 0)
if abs(old_gross - new_gross) > 0.01:
return True
old_net = float(old_rate.get('rate_net') or 0)
new_net = float(new_rate.get('net_rate') or 0)
if abs(old_net - new_net) > 0.01:
return True
# Compare tariff availability
old_tariffs = old_rate.get('tariffs_data', {})
if isinstance(old_tariffs, str):
try:
old_tariffs = json.loads(old_tariffs)
except json.JSONDecodeError:
old_tariffs = {}
new_tariffs = new_rate.get('tariffs_data', {})
old_tariff_list = old_tariffs.get('tariffs', [])
new_tariff_list = new_tariffs.get('tariffs', [])
# Different number of tariffs
if len(old_tariff_list) != len(new_tariff_list):
return True
# Compare each tariff's key attributes
for old_t, new_t in zip(old_tariff_list, new_tariff_list):
# Name changed
if old_t.get('name') != new_t.get('name'):
return True
# Availability status changed
if old_t.get('success') != new_t.get('success'):
return True
# Rate changed significantly
old_rate_val = float(old_t.get('rate') or 0)
new_rate_val = float(new_t.get('rate') or 0)
if abs(old_rate_val - new_rate_val) > 0.01:
return True
# Min stay changed
if old_t.get('min_stay') != new_t.get('min_stay'):
return True
# Multi-night availability changed
if old_t.get('available_for_min_stay') != new_t.get('available_for_min_stay'):
return True
return False
def save_rate_snapshot(db, category_id: str, rate_date: date, rate: Dict) -> str:
"""
Save rate to database using snapshot logic.
If rate has changed from latest version, insert new row.
If rate is the same, just update last_verified_at.
Returns: 'inserted', 'verified', or 'error'
"""
gross_rate = rate.get('gross_rate')
net_rate = rate.get('net_rate')
tariffs_data = rate.get('tariffs_data', {})
# Get the latest rate for this category/date
existing = db.execute(
text("""
SELECT id, rate_gross, rate_net, tariffs_data
FROM newbook_current_rates
WHERE category_id = :category_id AND rate_date = :rate_date
ORDER BY valid_from DESC
LIMIT 1
"""),
{"category_id": category_id, "rate_date": rate_date}
).fetchone()
if existing:
existing_dict = {
'rate_gross': existing.rate_gross,
'rate_net': existing.rate_net,
'tariffs_data': existing.tariffs_data
}
else:
existing_dict = None
if rates_changed(existing_dict, rate):
# Rates changed - insert new snapshot
db.execute(
text("""
INSERT INTO newbook_current_rates
(category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from, last_verified_at)
VALUES (:category_id, :rate_date, :rate_gross, :rate_net,
CAST(:tariffs_data AS jsonb), NOW(), NOW())
"""),
{
"category_id": category_id,
"rate_date": rate_date,
"rate_gross": gross_rate,
"rate_net": net_rate,
"tariffs_data": json.dumps(tariffs_data)
}
)
return 'inserted'
else:
# Rates unchanged - just verify
db.execute(
text("""
UPDATE newbook_current_rates
SET last_verified_at = NOW()
WHERE id = :id
"""),
{"id": existing.id}
)
return 'verified'
def needs_multi_night_check(tariff: Dict, days_ahead: int) -> Optional[int]:
"""
Check if a tariff needs multi-night verification.
Returns the min_stay value if a multi-night check is needed, None otherwise.
Skips tariffs with advance booking restrictions that aren't met.
"""
min_stay = tariff.get('min_stay')
if not min_stay or min_stay <= 1:
return None
if tariff.get('success', False):
return None # Already available as single-night, no recheck needed
# Check for advance booking requirement
message = tariff.get('message', '') or ''
advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
if advance_match:
min_advance_days = int(advance_match.group(1))
if days_ahead < min_advance_days:
return None # Within advance period - recheck won't help
return min_stay
async def run_fetch_current_rates(horizon_days: int = 720, start_date: date = None):
"""
Fetch current rates for all included categories and store in database.
Args:
horizon_days: Number of days ahead to fetch (default 720 for scheduled, configurable for manual)
start_date: Start date for fetch (default today)
Processing: Day-by-day with progressive commits.
For each date:
1. Fetch single-night rates (all categories in one API call)
2. Check if any tariffs need multi-night verification
3. If so, run multi-night check immediately for that date
4. Save all rates for that date to DB
5. Commit every COMMIT_BATCH_SIZE days
This means if the job fails at day 400, the first 390+ days are already saved.
"""
logger.info(f"Starting current rates fetch ({horizon_days} days)")
db = next(iter([SyncSessionLocal()]))
today = start_date or date.today()
try:
# Get VAT rate from config
vat_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
).fetchone()
vat_rate_str = vat_result.config_value if vat_result and vat_result.config_value else '0.20'
# Get all included room categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = set(row.site_id for row in cat_result.fetchall())
if not included_categories:
logger.warning("No included room categories found")
return
logger.info(f"Fetching rates for {len(included_categories)} categories")
# Import rates client
import base64
from services.newbook_rates_client import NewbookRatesClient
# Get credentials from config (decrypt encrypted values)
config_result = db.execute(
text("""
SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted
FROM system_config
WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region')
""")
)
config = {}
for row in config_result.fetchall():
value = row.config_value
if row.is_encrypted and value:
try:
value = base64.b64decode(value.encode()).decode()
except Exception:
pass # Use raw value if decryption fails
config[row.config_key] = value
if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
logger.error("Newbook credentials not configured")
return
# Create client
client = NewbookRatesClient(
api_key=config['newbook_api_key'],
username=config['newbook_username'],
password=config['newbook_password'],
region=config['newbook_region'],
vat_rate=Decimal(vat_rate_str)
)
async with client:
inserted_total = 0
verified_total = 0
multi_night_checks = 0
skipped_advance = 0
current_date = today
day_count = 0
while current_date <= today + timedelta(days=horizon_days):
day_count += 1
days_ahead = (current_date - today).days
try:
# Step 1: Fetch single-night rates for all categories on this date
day_rates = await client.fetch_single_date_all_categories(
current_date, guests_adults=2, guests_children=0
)
# Step 2: Check for multi-night verification needs and run inline
# Collect unique min_stay values needed for this date
nights_needed: Set[int] = set()
for cat_id, rates in day_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
check = needs_multi_night_check(tariff, days_ahead)
if check:
nights_needed.add(check)
elif tariff.get('min_stay') and tariff['min_stay'] > 1 and not tariff.get('success', False):
skipped_advance += 1
# Step 3: Run multi-night checks for this date if needed
multi_night_results: Dict[int, Dict[str, Dict[str, bool]]] = {}
for nights in sorted(nights_needed):
try:
result = await client.fetch_multi_night_for_date(
current_date, nights
)
multi_night_results[nights] = result
multi_night_checks += 1
await asyncio.sleep(1.0) # Rate limiting
except Exception as e:
logger.warning(f"Multi-night check failed for {current_date} ({nights}n): {e}")
# Step 4: Update tariffs with multi-night results and save to DB
for cat_id, rates in day_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
tariffs_data = rate.get('tariffs_data', {})
# Apply multi-night results to tariffs
for tariff in tariffs_data.get('tariffs', []):
min_stay = tariff.get('min_stay')
if min_stay and min_stay > 1 and min_stay in multi_night_results:
cat_availability = multi_night_results[min_stay].get(cat_id, {})
tariff_name = tariff.get('name', '')
tariff['available_for_min_stay'] = cat_availability.get(tariff_name, False)
# Save to DB
result = save_rate_snapshot(db, cat_id, current_date, rate)
if result == 'inserted':
inserted_total += 1
elif result == 'verified':
verified_total += 1
if day_count % 50 == 0 or nights_needed:
logger.info(
f"Day {day_count}/{horizon_days}: {current_date}"
f" | {inserted_total} new, {verified_total} verified"
f"{f' | {len(nights_needed)} multi-night checks' if nights_needed else ''}"
)
except Exception as e:
logger.warning(f"Failed to fetch rates for {current_date}: {e}")
# Step 5: Commit periodically
if day_count % COMMIT_BATCH_SIZE == 0:
db.commit()
current_date += timedelta(days=1)
await asyncio.sleep(1.0) # Rate limiting between days
# Final commit for remaining days
db.commit()
if skipped_advance > 0:
logger.info(f"Skipped {skipped_advance} multi-night checks (advance booking restriction)")
logger.info(
f"Complete: {inserted_total} new snapshots, {verified_total} verified unchanged, "
f"{multi_night_checks} multi-night checks"
)
logger.info("Current rates fetch completed successfully")
except Exception as e:
logger.error(f"Current rates fetch failed: {e}")
db.rollback()
raise
finally:
db.close()

276
backend/jobs/forecast_daily.py Normal file
View file

@ -0,0 +1,276 @@
"""
Daily forecast generation job
Runs Prophet, XGBoost, and Pickup models
"""
import json
import logging
import uuid
from datetime import date, timedelta
from typing import List, Optional
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
async def run_daily_forecast(
horizon_days: int = 14,
start_days: int = 0,
models: Optional[List[str]] = None,
triggered_by: str = "scheduler"
):
"""
Run daily forecast update for specified horizon.
Args:
horizon_days: How many days ahead to forecast
start_days: Start from N days in the future (for medium/long term)
models: Which models to run (default: all)
triggered_by: Who triggered this run
"""
if models is None:
models = ['prophet', 'xgboost', 'pickup', 'catboost']
run_id = str(uuid.uuid4())
forecast_from = date.today() + timedelta(days=start_days)
forecast_to = date.today() + timedelta(days=horizon_days)
logger.info(f"Starting forecast run {run_id}: {forecast_from} to {forecast_to}, models: {models}")
db = next(iter([SyncSessionLocal()]))
try:
# Log run start
db.execute(
text("""
INSERT INTO forecast_runs (
run_id, run_type, started_at, status,
forecast_from, forecast_to, models_run, triggered_by
) VALUES (
:run_id, 'scheduled', NOW(), 'running',
:forecast_from, :forecast_to, :models, :triggered_by
)
"""),
{
"run_id": run_id,
"forecast_from": forecast_from,
"forecast_to": forecast_to,
"models": json.dumps(models),
"triggered_by": triggered_by
}
)
db.commit()
# Get metrics to forecast
result = db.execute(
text("""
SELECT metric_code, use_prophet, use_xgboost, use_pickup,
COALESCE(use_catboost, TRUE) as use_catboost
FROM forecast_metrics
WHERE is_active = TRUE
""")
)
metrics = result.fetchall()
forecasts_generated = 0
for metric in metrics:
metric_code = metric.metric_code
# Run Prophet if applicable
if 'prophet' in models and metric.use_prophet:
try:
from services.forecasting.prophet_model import run_prophet_forecast
prophet_forecasts = await run_prophet_forecast(
db, metric_code, forecast_from, forecast_to
)
forecasts_generated += len(prophet_forecasts)
except Exception as e:
logger.error(f"Prophet forecast failed for {metric_code}: {e}")
db.rollback() # Rollback failed transaction
# Run XGBoost if applicable
if 'xgboost' in models and metric.use_xgboost:
try:
from services.forecasting.xgboost_model import run_xgboost_forecast
xgboost_forecasts = await run_xgboost_forecast(
db, metric_code, forecast_from, forecast_to
)
forecasts_generated += len(xgboost_forecasts)
except Exception as e:
logger.error(f"XGBoost forecast failed for {metric_code}: {e}")
db.rollback() # Rollback failed transaction
# Run Pickup if applicable (only for short-term)
if 'pickup' in models and metric.use_pickup and start_days < 30:
try:
from services.forecasting.pickup_model import run_pickup_forecast
pickup_forecasts = await run_pickup_forecast(
db, metric_code, forecast_from, forecast_to
)
forecasts_generated += len(pickup_forecasts)
except Exception as e:
logger.error(f"Pickup forecast failed for {metric_code}: {e}")
db.rollback() # Rollback failed transaction
# Run CatBoost if applicable
if 'catboost' in models and getattr(metric, 'use_catboost', True):
try:
from services.forecasting.catboost_model import run_catboost_forecast
catboost_forecasts = await run_catboost_forecast(
db, metric_code, forecast_from, forecast_to
)
forecasts_generated += len(catboost_forecasts)
except Exception as e:
logger.error(f"CatBoost forecast failed for {metric_code}: {e}")
db.rollback() # Rollback failed transaction
# Run blended model (accuracy-weighted average of prophet, xgboost, catboost)
if 'blended' in models:
try:
logger.info("Generating blended forecasts with accuracy-based weighting")
# Get accuracy scores for model weighting (from last 90 days)
# Calculate weights per metric
metric_weights = {}
for metric in metrics:
metric_code = metric.metric_code
try:
accuracy_result = db.execute(
text("""
SELECT
AVG(ABS(prophet_pct_error)) as prophet_mape,
AVG(ABS(xgboost_pct_error)) as xgboost_mape,
AVG(ABS(catboost_pct_error)) as catboost_mape
FROM actual_vs_forecast
WHERE date >= CURRENT_DATE - INTERVAL '90 days'
AND date < CURRENT_DATE
AND metric_type = :metric
AND actual_value IS NOT NULL
"""),
{"metric": metric_code}
)
accuracy_row = accuracy_result.fetchone()
# Calculate inverse-MAPE weights (lower MAPE = higher weight)
if accuracy_row and accuracy_row.prophet_mape and accuracy_row.xgboost_mape and accuracy_row.catboost_mape:
prophet_mape = float(accuracy_row.prophet_mape) or 10
xgboost_mape = float(accuracy_row.xgboost_mape) or 10
catboost_mape = float(accuracy_row.catboost_mape) or 10
inv_prophet = 1 / max(prophet_mape, 0.1)
inv_xgboost = 1 / max(xgboost_mape, 0.1)
inv_catboost = 1 / max(catboost_mape, 0.1)
total_inv = inv_prophet + inv_xgboost + inv_catboost
metric_weights[metric_code] = {
'prophet': inv_prophet / total_inv,
'xgboost': inv_xgboost / total_inv,
'catboost': inv_catboost / total_inv
}
else:
# Equal weights if no accuracy data
metric_weights[metric_code] = {'prophet': 1/3, 'xgboost': 1/3, 'catboost': 1/3}
except Exception:
# Default to equal weights on error
metric_weights[metric_code] = {'prophet': 1/3, 'xgboost': 1/3, 'catboost': 1/3}
# Get all forecasts from the three models for this run
result = db.execute(
text("""
SELECT forecast_date, forecast_type, model_type, predicted_value
FROM forecasts
WHERE run_id = :run_id
AND model_type IN ('prophet', 'xgboost', 'catboost')
ORDER BY forecast_date, forecast_type
"""),
{"run_id": run_id}
)
rows = result.fetchall()
if rows:
# Group by forecast_date and forecast_type
forecasts_by_date_type = {}
for row in rows:
key = (row.forecast_date, row.forecast_type)
if key not in forecasts_by_date_type:
forecasts_by_date_type[key] = {}
forecasts_by_date_type[key][row.model_type] = float(row.predicted_value)
# Calculate weighted blended forecast for each date/type combination
blended_count = 0
for (forecast_date, forecast_type), model_forecasts in forecasts_by_date_type.items():
# Only blend if we have at least 2 models
if len(model_forecasts) >= 2:
# Get weights for this metric
weights = metric_weights.get(forecast_type, {'prophet': 1/3, 'xgboost': 1/3, 'catboost': 1/3})
# Calculate weighted average
weighted_sum = 0
weight_total = 0
for model, value in model_forecasts.items():
weight = weights.get(model, 0)
weighted_sum += value * weight
weight_total += weight
blended_value = weighted_sum / weight_total if weight_total > 0 else sum(model_forecasts.values()) / len(model_forecasts)
# Insert blended forecast
db.execute(
text("""
INSERT INTO forecasts
(run_id, forecast_date, forecast_type, model_type, predicted_value, generated_at)
VALUES
(:run_id, :forecast_date, :forecast_type, 'blended', :predicted_value, NOW())
"""),
{
"run_id": run_id,
"forecast_date": forecast_date,
"forecast_type": forecast_type,
"predicted_value": round(blended_value, 2)
}
)
blended_count += 1
db.commit()
forecasts_generated += blended_count
logger.info(f"Generated {blended_count} accuracy-weighted blended forecasts")
else:
logger.warning("No individual model forecasts found for blending")
except Exception as e:
logger.error(f"Blended forecast generation failed: {e}")
# Update run status
db.execute(
text("""
UPDATE forecast_runs
SET completed_at = NOW(), status = 'success'
WHERE run_id = :run_id
"""),
{"run_id": run_id}
)
db.commit()
logger.info(f"Forecast run {run_id} completed: {forecasts_generated} forecasts generated")
except Exception as e:
logger.error(f"Forecast run {run_id} failed: {e}")
# Rollback the failed transaction first
db.rollback()
try:
db.execute(
text("""
UPDATE forecast_runs
SET completed_at = NOW(), status = 'failed', error_message = :error
WHERE run_id = :run_id
"""),
{"run_id": run_id, "error": str(e)}
)
db.commit()
except Exception as update_error:
logger.error(f"Failed to update error status: {update_error}")
raise
finally:
db.close()

150
backend/jobs/metrics_aggregation.py Normal file
View file

@ -0,0 +1,150 @@
"""
Metrics aggregation job for forecast_data database
Populates daily_metrics from newbook_bookings_stats
This is the data source for forecasting models (Prophet, XGBoost, CatBoost).
"""
import logging
from datetime import date, timedelta
from typing import List, Optional
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
async def run_metrics_aggregation(
from_date: Optional[date] = None,
to_date: Optional[date] = None
):
"""
Populate daily_metrics table from newbook_bookings_stats.
This provides the historical actuals needed for forecasting models.
Args:
from_date: Start date (defaults to 2 years ago)
to_date: End date (defaults to yesterday)
"""
logger.info("Starting metrics aggregation job")
db = next(iter([SyncSessionLocal()]))
try:
# Default date range: 2 years of history
if from_date is None:
from_date = date.today() - timedelta(days=730)
if to_date is None:
to_date = date.today() - timedelta(days=1)
logger.info(f"Aggregating metrics from {from_date} to {to_date}")
# Get data from newbook_bookings_stats
result = db.execute(
text("""
SELECT
date,
booking_count, -- room nights (occupied rooms)
total_occupancy_pct, -- occupancy percentage
guests_count, -- total guests
adults_count,
children_count,
rooms_count, -- available rooms
bookable_count -- bookable rooms (rooms - maintenance)
FROM newbook_bookings_stats
WHERE date BETWEEN :from_date AND :to_date
ORDER BY date
"""),
{"from_date": from_date, "to_date": to_date}
)
stats_rows = result.fetchall()
if not stats_rows:
logger.warning("No data found in newbook_bookings_stats")
return
logger.info(f"Found {len(stats_rows)} days of data to aggregate")
# Metrics to populate
metrics_count = 0
for row in stats_rows:
d = row.date
# Define metrics from newbook_bookings_stats
metrics_to_insert = []
# Room nights (occupied rooms)
if row.booking_count is not None:
metrics_to_insert.append(("hotel_room_nights", row.booking_count))
# Occupancy percentage
if row.total_occupancy_pct is not None:
metrics_to_insert.append(("hotel_occupancy_pct", float(row.total_occupancy_pct)))
# Guest count
if row.guests_count is not None:
metrics_to_insert.append(("hotel_guests", row.guests_count))
# Insert/update all metrics
for metric_code, actual_value in metrics_to_insert:
db.execute(
text("""
INSERT INTO daily_metrics (date, metric_code, actual_value, source, updated_at)
VALUES (:date, :metric_code, :actual_value, 'newbook', NOW())
ON CONFLICT (date, metric_code) DO UPDATE SET
actual_value = :actual_value,
updated_at = NOW()
"""),
{
"date": d,
"metric_code": metric_code,
"actual_value": actual_value
}
)
metrics_count += 1
db.commit()
logger.info(f"Aggregated {metrics_count} metric records from {len(stats_rows)} days")
except Exception as e:
logger.error(f"Metrics aggregation failed: {e}")
db.rollback()
raise
finally:
db.close()
async def backfill_daily_metrics():
"""
Backfill all available history from newbook_bookings_stats to daily_metrics.
Call this once when setting up forecasting on forecast_data database.
"""
logger.info("Starting full backfill of daily_metrics")
db = next(iter([SyncSessionLocal()]))
try:
# Find the earliest date in newbook_bookings_stats
result = db.execute(text("SELECT MIN(date) as min_date FROM newbook_bookings_stats"))
row = result.fetchone()
if not row or not row.min_date:
logger.warning("No data in newbook_bookings_stats to backfill")
return
from_date = row.min_date
to_date = date.today() - timedelta(days=1)
logger.info(f"Backfilling from {from_date} to {to_date}")
await run_metrics_aggregation(from_date=from_date, to_date=to_date)
logger.info("Backfill completed successfully")
except Exception as e:
logger.error(f"Backfill failed: {e}")
raise
finally:
db.close()

439
backend/jobs/pace_snapshot_v2.py Normal file
View file

@ -0,0 +1,439 @@
"""
Pace Snapshot V2 - Enhanced pace capture for pickup-v2 model
Captures:
1. Per-category room counts at each lead time (category_booking_pace)
2. Total booked accommodation revenue at each lead time (revenue_pace)
This job runs alongside the existing pickup_snapshot job.
Uses 364-day offset for prior year comparison (52 weeks = day-of-week alignment).
"""
import logging
from datetime import date, timedelta
from decimal import Decimal
from typing import Dict, List, Optional
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
# Valid booking statuses for aggregation
VALID_STATUSES = ('Unconfirmed', 'Confirmed', 'Arrived', 'Departed')
# All tracked pace intervals (same as booking_pace table structure)
PACE_INTERVALS = [
# Monthly (months 7-12)
365, 330, 300, 270, 240, 210,
# Weekly (weeks 5-25)
177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
# Daily (days 0-30)
30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
]
def get_lead_time_column(lead_days: int) -> str:
"""
Map lead days to the appropriate column in pace tables.
Uses round-up logic for days between tracked intervals.
"""
if lead_days <= 0:
return "d0"
elif lead_days <= 30:
return f"d{lead_days}"
elif lead_days <= 177:
# Weekly intervals - find next higher
weekly_cols = [37, 44, 51, 58, 65, 72, 79, 86, 93, 100, 107, 114, 121, 128, 135, 142, 149, 156, 163, 170, 177]
for col in weekly_cols:
if lead_days <= col:
return f"d{col}"
return "d177"
else:
# Monthly intervals
monthly_cols = [210, 240, 270, 300, 330, 365]
for col in monthly_cols:
if lead_days <= col:
return f"d{col}"
return "d365"
def get_rate_for_date(raw_json: dict, target_date: date, vat_rate: Decimal) -> Decimal:
"""
Extract net accommodation rate from tariffs_quoted for a specific stay_date.
Returns net amount (after VAT deduction).
"""
if not raw_json:
return Decimal('0')
tariffs = raw_json.get("tariffs_quoted", [])
target_str = target_date.strftime("%Y-%m-%d")
for tariff in tariffs:
if tariff.get("stay_date") == target_str:
charge_amount = Decimal(str(tariff.get("charge_amount", 0) or 0))
# Try to get net from taxes array if available
taxes = tariff.get("taxes", [])
if taxes and charge_amount > 0:
tax_amount = sum(Decimal(str(t.get("amount", 0) or 0)) for t in taxes)
net_amount = charge_amount - tax_amount
else:
# Fallback: calculate net using VAT rate
net_amount = charge_amount / (1 + vat_rate)
return net_amount
return Decimal('0')
async def run_pace_snapshot_v2():
"""
Capture per-category room counts and total revenue at each lead time.
Updates:
- category_booking_pace: room counts by category for each future date
- revenue_pace: total booked accommodation revenue for each future date
"""
logger.info("Starting pace snapshot v2 capture")
db = next(iter([SyncSessionLocal()]))
today = date.today()
try:
# Get accommodation VAT rate from config
vat_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
).fetchone()
vat_rate = Decimal(vat_result.config_value) if vat_result and vat_result.config_value else Decimal('0.20')
# Get all included room categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = [row.site_id for row in cat_result.fetchall()]
if not included_categories:
logger.warning("No included room categories found")
return
# Process each tracked interval
for interval in PACE_INTERVALS:
stay_date = today + timedelta(days=interval)
column_name = f"d{interval}"
# === 1. Capture per-category room counts ===
cat_counts = await capture_category_counts(db, stay_date, included_categories)
for category_id, count in cat_counts.items():
db.execute(
text(f"""
INSERT INTO category_booking_pace (arrival_date, category_id, {column_name}, updated_at)
VALUES (:stay_date, :category_id, :count, NOW())
ON CONFLICT (arrival_date, category_id) DO UPDATE
SET {column_name} = :count, updated_at = NOW()
"""),
{"stay_date": stay_date, "category_id": category_id, "count": count}
)
# === 2. Capture total booked revenue ===
total_revenue = await capture_booked_revenue(db, stay_date, vat_rate, included_categories)
db.execute(
text(f"""
INSERT INTO revenue_pace (stay_date, {column_name}, updated_at)
VALUES (:stay_date, :revenue, NOW())
ON CONFLICT (stay_date) DO UPDATE
SET {column_name} = :revenue, updated_at = NOW()
"""),
{"stay_date": stay_date, "revenue": float(total_revenue)}
)
# Also fill gap dates (31-36, 38-43, etc.) with their bracketed column
await fill_gap_dates(db, today, vat_rate, included_categories)
db.commit()
logger.info(f"Pace snapshot v2 completed for {today}")
except Exception as e:
logger.error(f"Pace snapshot v2 failed: {e}")
db.rollback()
raise
finally:
db.close()
async def capture_category_counts(db, stay_date: date, included_categories: List[str]) -> Dict[str, int]:
"""
Count rooms booked per category for a given stay date.
Returns dict of {category_id: count}
"""
result = db.execute(
text("""
SELECT category_id, COUNT(*) as count
FROM newbook_bookings_data
WHERE arrival_date <= :stay_date
AND departure_date > :stay_date
AND status IN :valid_statuses
AND category_id IN :categories
GROUP BY category_id
"""),
{"stay_date": stay_date, "valid_statuses": VALID_STATUSES, "categories": tuple(included_categories)}
)
counts = {cat: 0 for cat in included_categories} # Initialize all categories with 0
for row in result.fetchall():
counts[row.category_id] = row.count
return counts
async def capture_booked_revenue(db, stay_date: date, vat_rate: Decimal, included_categories: List[str]) -> Decimal:
"""
Calculate total booked accommodation revenue (net) for a given stay date.
Sums up tariffs from all active bookings that span this date.
"""
result = db.execute(
text("""
SELECT raw_json
FROM newbook_bookings_data
WHERE arrival_date <= :stay_date
AND departure_date > :stay_date
AND status IN :valid_statuses
AND category_id IN :categories
"""),
{"stay_date": stay_date, "valid_statuses": VALID_STATUSES, "categories": tuple(included_categories)}
)
total_revenue = Decimal('0')
for row in result.fetchall():
if row.raw_json:
revenue = get_rate_for_date(row.raw_json, stay_date, vat_rate)
total_revenue += revenue
return total_revenue
async def fill_gap_dates(db, today: date, vat_rate: Decimal, included_categories: List[str]):
"""
Fill gap dates (between tracked intervals) with their bracketed column value.
These dates fall between weekly intervals and need the next higher column updated.
"""
gap_updates = 0
for days_out in range(31, 90): # Cover the gap range where intervals are weekly
if days_out in PACE_INTERVALS:
continue # Already handled in main loop
stay_date = today + timedelta(days=days_out)
bracket_col = get_lead_time_column(days_out)
# Capture category counts
cat_counts = await capture_category_counts(db, stay_date, included_categories)
for category_id, count in cat_counts.items():
db.execute(
text(f"""
INSERT INTO category_booking_pace (arrival_date, category_id, {bracket_col}, updated_at)
VALUES (:stay_date, :category_id, :count, NOW())
ON CONFLICT (arrival_date, category_id) DO UPDATE
SET {bracket_col} = :count, updated_at = NOW()
"""),
{"stay_date": stay_date, "category_id": category_id, "count": count}
)
# Capture revenue
total_revenue = await capture_booked_revenue(db, stay_date, vat_rate, included_categories)
db.execute(
text(f"""
INSERT INTO revenue_pace (stay_date, {bracket_col}, updated_at)
VALUES (:stay_date, :revenue, NOW())
ON CONFLICT (stay_date) DO UPDATE
SET {bracket_col} = :revenue, updated_at = NOW()
"""),
{"stay_date": stay_date, "revenue": float(total_revenue)}
)
gap_updates += 1
logger.info(f"Filled {gap_updates} gap dates for category pace and revenue pace")
async def backfill_pace_v2(db=None):
"""
Backfill historical pace v2 data using booking_placed timestamps.
Reconstructs what category counts and revenue would have been at each lead time
for historical dates.
"""
import sys
print("[PACE-V2-BACKFILL] Starting backfill...", flush=True)
close_db = False
if db is None:
db = next(iter([SyncSessionLocal()]))
close_db = True
try:
# Get VAT rate
vat_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
).fetchone()
vat_rate = Decimal(vat_result.config_value) if vat_result and vat_result.config_value else Decimal('0.20')
# Get included categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = [row.site_id for row in cat_result.fetchall()]
if not included_categories:
print("[PACE-V2-BACKFILL] No included categories found", flush=True)
return
# Get all unique stay dates from bookings_stats
result = db.execute(
text("""
SELECT date as stay_date
FROM newbook_bookings_stats
WHERE date >= CURRENT_DATE - INTERVAL '2 years'
ORDER BY date
""")
)
stay_dates = [row.stay_date for row in result.fetchall()]
print(f"[PACE-V2-BACKFILL] Found {len(stay_dates)} dates to process", flush=True)
today = date.today()
for i, stay_date in enumerate(stay_dates):
if i % 100 == 0:
print(f"[PACE-V2-BACKFILL] Processing: {i}/{len(stay_dates)} dates...", flush=True)
db.commit()
await backfill_pace_v2_for_date(db, stay_date, today, vat_rate, included_categories)
db.commit()
print(f"[PACE-V2-BACKFILL] Complete: {len(stay_dates)} dates processed", flush=True)
except Exception as e:
print(f"[PACE-V2-BACKFILL] FAILED: {e}", flush=True)
db.rollback()
raise
finally:
if close_db:
db.close()
async def backfill_pace_v2_for_date(
db,
stay_date: date,
today: date,
vat_rate: Decimal,
included_categories: List[str]
):
"""
Backfill pace v2 data for a single date using booking_placed timestamps.
"""
pace_category_values: Dict[str, Dict[str, int]] = {cat: {} for cat in included_categories}
pace_revenue_values: Dict[str, Decimal] = {}
for interval in PACE_INTERVALS:
snapshot_date = stay_date - timedelta(days=interval)
if snapshot_date > today:
continue # This snapshot hasn't happened yet
if snapshot_date < date(2020, 1, 1):
continue # Don't go too far back
column_name = f"d{interval}"
# Count per-category bookings that existed at snapshot_date
result = db.execute(
text("""
SELECT category_id, COUNT(*) as count
FROM newbook_bookings_data
WHERE arrival_date <= :stay_date
AND departure_date > :stay_date
AND status IN :valid_statuses
AND category_id IN :categories
AND booking_placed IS NOT NULL
AND booking_placed::date <= :snapshot_date
GROUP BY category_id
"""),
{
"stay_date": stay_date,
"valid_statuses": VALID_STATUSES,
"categories": tuple(included_categories),
"snapshot_date": snapshot_date
}
)
for row in result.fetchall():
pace_category_values[row.category_id][column_name] = row.count
# Calculate revenue that was booked at snapshot_date
result = db.execute(
text("""
SELECT raw_json
FROM newbook_bookings_data
WHERE arrival_date <= :stay_date
AND departure_date > :stay_date
AND status IN :valid_statuses
AND category_id IN :categories
AND booking_placed IS NOT NULL
AND booking_placed::date <= :snapshot_date
"""),
{
"stay_date": stay_date,
"valid_statuses": VALID_STATUSES,
"categories": tuple(included_categories),
"snapshot_date": snapshot_date
}
)
total_revenue = Decimal('0')
for row in result.fetchall():
if row.raw_json:
revenue = get_rate_for_date(row.raw_json, stay_date, vat_rate)
total_revenue += revenue
pace_revenue_values[column_name] = total_revenue
# Upsert category pace values
for category_id, columns in pace_category_values.items():
if not columns:
continue
col_names = list(columns.keys())
set_clauses = ", ".join([f"{col} = :{col}" for col in col_names])
insert_cols = ", ".join(col_names)
insert_vals = ", ".join([f":{col}" for col in col_names])
db.execute(
text(f"""
INSERT INTO category_booking_pace (arrival_date, category_id, {insert_cols}, updated_at)
VALUES (:stay_date, :category_id, {insert_vals}, NOW())
ON CONFLICT (arrival_date, category_id) DO UPDATE SET
{set_clauses}, updated_at = NOW()
"""),
{"stay_date": stay_date, "category_id": category_id, **columns}
)
# Upsert revenue pace values
if pace_revenue_values:
col_names = list(pace_revenue_values.keys())
float_values = {k: float(v) for k, v in pace_revenue_values.items()}
set_clauses = ", ".join([f"{col} = :{col}" for col in col_names])
insert_cols = ", ".join(col_names)
insert_vals = ", ".join([f":{col}" for col in col_names])
db.execute(
text(f"""
INSERT INTO revenue_pace (stay_date, {insert_cols}, updated_at)
VALUES (:stay_date, {insert_vals}, NOW())
ON CONFLICT (stay_date) DO UPDATE SET
{set_clauses}, updated_at = NOW()
"""),
{"stay_date": stay_date, **float_values}
)

208
backend/jobs/pickup_snapshot.py Normal file
View file

@ -0,0 +1,208 @@
"""
Pickup snapshot job - captures daily on-the-books values
Used by the pickup model for pace comparison
Prior year comparison uses 364 days (52 weeks) for day-of-week alignment:
- Monday compares to Monday
- Saturday compares to Saturday
"""
import logging
from datetime import date, timedelta
from sqlalchemy import text
from database import SyncSessionLocal
from utils.time_alignment import get_prior_year_daily, SQL_PRIOR_YEAR_OFFSET
logger = logging.getLogger(__name__)
async def run_pickup_snapshot():
"""
Capture daily on-the-books snapshot for future dates.
Stores OTB values at various lead times for pickup model.
"""
logger.info("Starting pickup snapshot capture")
db = next(iter([SyncSessionLocal()]))
snapshot_date = date.today()
# Overflow room category (category_id=5) is used for chargeable no-shows/cancellations
# and should be excluded from room night counts
overflow_category_id = '5'
try:
# Capture OTB for next 365 days (extended from 60 for longer-term forecasting)
for days_out in range(1, 366):
stay_date = snapshot_date + timedelta(days=days_out)
# Get total available rooms for this date from occupancy report
rooms_result = db.execute(
text("""
SELECT COALESCE(SUM(available), 25) as total_rooms
FROM newbook_occupancy_report
WHERE date = :stay_date
"""),
{"stay_date": stay_date}
).fetchone()
total_rooms = rooms_result.total_rooms if rooms_result and rooms_result.total_rooms else 25
# Get hotel occupancy OTB (count rooms on the books for this stay date)
# EXCLUDES overflow category (chargeable no-shows)
hotel_result = db.execute(
text("""
SELECT COUNT(DISTINCT newbook_id) as bookings,
SUM(CASE WHEN LOWER(status) IN ('confirmed', 'provisional', 'unconfirmed', 'arrived') THEN 1 ELSE 0 END) as active_bookings
FROM newbook_bookings
WHERE arrival_date <= :stay_date AND departure_date > :stay_date
AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')
AND (category_id IS NULL OR category_id != :overflow_cat)
"""),
{"stay_date": stay_date, "overflow_cat": overflow_category_id}
)
hotel_row = hotel_result.fetchone()
hotel_otb = hotel_row.active_bookings or 0
# Get dinner covers OTB
dinner_result = db.execute(
text("""
SELECT COUNT(*) as bookings, COALESCE(SUM(covers), 0) as total_covers
FROM resos_bookings
WHERE booking_date = :stay_date
AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show')
AND booking_time >= '15:00'
"""),
{"stay_date": stay_date}
)
dinner_row = dinner_result.fetchone()
dinner_otb = dinner_row.total_covers or 0
# Get lunch covers OTB
lunch_result = db.execute(
text("""
SELECT COUNT(*) as bookings, COALESCE(SUM(covers), 0) as total_covers
FROM resos_bookings
WHERE booking_date = :stay_date
AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show')
AND booking_time < '15:00'
"""),
{"stay_date": stay_date}
)
lunch_row = lunch_result.fetchone()
lunch_otb = lunch_row.total_covers or 0
# Get prior year comparison data (same day of week, exactly 52 weeks ago)
# Uses SQL_PRIOR_YEAR_OFFSET (364 days = 52 weeks) for Mon→Mon, Sat→Sat alignment
prior_year_stay_date = get_prior_year_daily(stay_date)
prior_year_snapshot_date = get_prior_year_daily(snapshot_date) # Same lead time last year
# Calculate prior year hotel OTB at same lead time using booking_placed
# EXCLUDES overflow category
prior_hotel_otb_result = db.execute(
text("""
SELECT COUNT(DISTINCT newbook_id) as otb_count
FROM newbook_bookings
WHERE arrival_date <= :prior_stay_date
AND departure_date > :prior_stay_date
AND LOWER(status) NOT IN ('cancelled', 'no show', 'no_show', 'quote', 'waitlist')
AND (raw_json->>'booking_placed')::timestamp <= :prior_snapshot_date
AND (category_id IS NULL OR category_id != :overflow_cat)
"""),
{"prior_stay_date": prior_year_stay_date, "prior_snapshot_date": prior_year_snapshot_date, "overflow_cat": overflow_category_id}
).fetchone()
prior_hotel_otb = prior_hotel_otb_result.otb_count if prior_hotel_otb_result else None
# Store snapshots for each metric
# For hotel_occupancy_pct: store room count, convert to % for otb_value
# For hotel_room_nights: store raw room count (no conversion)
# For restaurant: store cover counts directly
for metric_type, otb_raw, otb_bookings_count in [
('hotel_occupancy_pct', hotel_otb, hotel_row.bookings),
('hotel_room_nights', hotel_otb, hotel_row.bookings), # Same count, stored as-is
('resos_dinner_covers', dinner_otb, dinner_row.bookings),
('resos_lunch_covers', lunch_otb, lunch_row.bookings)
]:
# Convert to percentage for occupancy metric, keep raw counts for others
if metric_type == 'hotel_occupancy_pct':
otb_value = (otb_raw / total_rooms) * 100 if total_rooms > 0 else 0
# Use 'is not None' check - 0 is valid data (no bookings at that lead time)
prior_otb_value = (prior_hotel_otb / total_rooms) * 100 if prior_hotel_otb is not None and total_rooms > 0 else None
elif metric_type == 'hotel_room_nights':
otb_value = otb_raw # Raw room count
prior_otb_value = prior_hotel_otb # Raw room count from prior year
else:
otb_value = otb_raw
prior_otb_value = None # Will try historical snapshots below
# Get prior year ACTUAL from daily_metrics (the final outcome)
prior_final_result = db.execute(
text("""
SELECT actual_value
FROM daily_metrics
WHERE date = :prior_date AND metric_code = :metric
"""),
{"prior_date": prior_year_stay_date, "metric": metric_type}
).fetchone()
prior_final = float(prior_final_result.actual_value) if prior_final_result and prior_final_result.actual_value else None
# Use reconstructed prior year OTB for hotel metrics, or try historical snapshots
if metric_type in ('hotel_occupancy_pct', 'hotel_room_nights') and prior_otb_value is not None:
prior_otb = prior_otb_value
else:
# For restaurant metrics, fall back to historical snapshots
prior_otb_result = db.execute(
text("""
SELECT otb_value
FROM pickup_snapshots
WHERE stay_date = :prior_stay_date
AND metric_type = :metric
AND days_out = :days_out
ORDER BY snapshot_date DESC LIMIT 1
"""),
{"prior_stay_date": prior_year_stay_date, "metric": metric_type, "days_out": days_out}
).fetchone()
# Use 'is not None' - 0 is valid OTB data
prior_otb = float(prior_otb_result.otb_value) if prior_otb_result and prior_otb_result.otb_value is not None else None
# Calculate pace vs prior year if we have comparison data
pace_pct = None
if prior_otb and prior_otb > 0:
pace_pct = ((otb_value - prior_otb) / prior_otb) * 100
db.execute(
text("""
INSERT INTO pickup_snapshots (
snapshot_date, stay_date, days_out, metric_type,
otb_value, otb_bookings, prior_year_otb, prior_year_final,
pace_vs_prior_pct, created_at
) VALUES (
:snapshot_date, :stay_date, :days_out, :metric_type,
:otb_value, :otb_bookings, :prior_year_otb, :prior_year_final,
:pace_pct, NOW()
)
ON CONFLICT (snapshot_date, stay_date, metric_type) DO UPDATE SET
otb_value = :otb_value,
prior_year_otb = :prior_year_otb,
pace_vs_prior_pct = :pace_pct
"""),
{
"snapshot_date": snapshot_date,
"stay_date": stay_date,
"days_out": days_out,
"metric_type": metric_type,
"otb_value": round(otb_value, 2), # % for hotel, covers for restaurant
"otb_bookings": otb_raw, # Raw count (rooms or covers)
"prior_year_otb": round(prior_otb, 2) if prior_otb is not None else None,
"prior_year_final": prior_final,
"pace_pct": round(pace_pct, 2) if pace_pct else None
}
)
db.commit()
logger.info(f"Pickup snapshot completed for {snapshot_date}")
except Exception as e:
logger.error(f"Pickup snapshot failed: {e}")
db.rollback()
raise
finally:
db.close()

476
backend/jobs/resos_aggregation.py Normal file
View file

@ -0,0 +1,476 @@
"""
Resos Bookings Aggregation Job
Aggregates resos_bookings_data into:
- resos_bookings_stats: daily aggregated stats with period/source breakdowns
- resos_booking_pace: lead-time snapshots for pickup forecasting (3 types)
"""
import json
import logging
from datetime import date, datetime, timedelta
from typing import Set, Dict, Any, Optional, Tuple, List
from collections import defaultdict
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
# Valid booking statuses for aggregation
# Note: Resos uses 'approved' for confirmed future reservations, 'left' for completed meals
VALID_STATUSES = ('approved', 'arrived', 'seated', 'left')
# All tracked pace intervals (same as Newbook)
PACE_INTERVALS = [
# Monthly (months 7-12)
365, 330, 300, 270, 240, 210,
# Weekly (weeks 5-25)
177, 170, 163, 156, 149, 142, 135, 128, 121, 114,
107, 100, 93, 86, 79, 72, 65, 58, 51, 44, 37,
# Daily (days 0-30)
30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
20, 19, 18, 17, 16, 15, 14, 13, 12, 11,
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
]
def parse_group_exclude_field(group_exclude_field: Optional[str], primary_booking_number: Optional[str]) -> Tuple[List[str], List[str]]:
"""
Parse group_exclude_field to extract linked bookings and exclude markers.
Returns: (all_booking_numbers, exclude_numbers)
"""
all_booking_numbers = []
exclude_numbers = []
# Always include primary booking number
if primary_booking_number:
all_booking_numbers.append(primary_booking_number)
if not group_exclude_field:
return all_booking_numbers, exclude_numbers
# Parse comma-separated entries
parts = group_exclude_field.split(',')
for part in parts:
part = part.strip()
if part.upper().startswith('NOT-#'):
# Exclude marker: NOT-#56748 → NB56748
booking_num = part[5:] # Remove "NOT-#"
exclude_numbers.append(f"NB{booking_num}")
elif part.startswith('#'):
# Additional booking: #12346 → NB12346
booking_num = part[1:] # Remove "#"
all_booking_numbers.append(f"NB{booking_num}")
return all_booking_numbers, exclude_numbers
async def aggregate_resos_bookings(triggered_by: str = "manual"):
"""
Aggregate Resos bookings into resos_bookings_stats.
Flow:
1. Find bookings changed since last_resos_aggregation_at
2. Calculate affected dates
3. Reaggregate each affected date
4. Update booking pace table (3 types)
5. Update last_resos_aggregation_at
"""
logger.info(f"Starting Resos bookings aggregation (triggered_by={triggered_by})")
db = SyncSessionLocal()
try:
# Get last aggregation timestamp
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'last_resos_aggregation_at'")
)
row = result.fetchone()
if row and row.config_value:
try:
last_ts = datetime.fromisoformat(row.config_value)
except ValueError:
last_ts = datetime.min
else:
last_ts = datetime.min
logger.info(f"Last aggregation: {last_ts}")
# Find bookings changed since last aggregation
result = db.execute(
text("""
SELECT resos_id, booking_date
FROM resos_bookings_data
WHERE fetched_at > :last_ts
"""),
{"last_ts": last_ts}
)
changed_bookings = result.fetchall()
if not changed_bookings:
logger.info("No changed bookings to aggregate")
# Still update pace table
await update_resos_booking_pace(db)
db.commit()
# Update timestamp
db.execute(
text("""
INSERT INTO system_config (config_key, config_value, updated_at)
VALUES ('last_resos_aggregation_at', :now, NOW())
ON CONFLICT (config_key) DO UPDATE SET
config_value = :now,
updated_at = NOW()
"""),
{"now": datetime.now().isoformat()}
)
db.commit()
return
logger.info(f"Found {len(changed_bookings)} changed bookings")
# Calculate affected dates
affected_dates: Set[date] = set()
for booking in changed_bookings:
if booking.booking_date:
affected_dates.add(booking.booking_date)
logger.info(f"Reaggregating {len(affected_dates)} affected dates")
# Aggregate each affected date
for target_date in sorted(affected_dates):
await aggregate_date(db, target_date)
# Update booking pace table (3 types)
await update_resos_booking_pace(db)
# Update last aggregation timestamp
db.execute(
text("""
INSERT INTO system_config (config_key, config_value, updated_at)
VALUES ('last_resos_aggregation_at', :now, NOW())
ON CONFLICT (config_key) DO UPDATE SET
config_value = :now,
updated_at = NOW()
"""),
{"now": datetime.now().isoformat()}
)
db.commit()
logger.info(f"Resos bookings aggregation completed: {len(affected_dates)} dates processed")
except Exception as e:
logger.error(f"Resos bookings aggregation failed: {e}", exc_info=True)
db.rollback()
raise
finally:
db.close()
async def aggregate_date(db, target_date: date):
"""
Aggregate all Resos bookings for a specific date into resos_bookings_stats.
"""
# Get all valid bookings for this date
result = db.execute(
text("""
SELECT
resos_id,
period_type,
covers,
source,
opening_hour_id,
is_hotel_guest,
is_dbb,
is_package,
total_guests,
hotel_booking_number,
group_exclude_field
FROM resos_bookings_data
WHERE booking_date = :target_date
AND status IN :valid_statuses
"""),
{"target_date": target_date, "valid_statuses": VALID_STATUSES}
)
bookings = result.fetchall()
# Initialize counters
breakfast_covers = 0
lunch_covers = 0
afternoon_covers = 0
dinner_covers = 0
other_covers = 0
breakfast_bookings = 0
lunch_bookings = 0
afternoon_bookings = 0
dinner_bookings = 0
other_bookings = 0
hotel_guest_covers = 0
non_hotel_guest_covers = 0
dbb_covers = 0
package_covers = 0
covers_by_source: Dict[str, int] = defaultdict(int)
covers_by_period: Dict[str, Dict[str, Any]] = {}
total_party_sizes = []
party_sizes_by_period: Dict[str, list] = defaultdict(list)
# Build hotel_booking_numbers mapping
hotel_booking_numbers: Dict[str, str] = {} # hotel_booking_number -> resos_id
bookings_with_hotel_link = 0
for booking in bookings:
period = booking.period_type or 'other'
covers = booking.covers or 0
source = booking.source or 'unknown'
opening_hour_id = booking.opening_hour_id
resos_id = booking.resos_id
# Count by period
if period == 'breakfast':
breakfast_covers += covers
breakfast_bookings += 1
elif period == 'lunch':
lunch_covers += covers
lunch_bookings += 1
elif period == 'afternoon':
afternoon_covers += covers
afternoon_bookings += 1
elif period == 'dinner':
dinner_covers += covers
dinner_bookings += 1
else:
other_covers += covers
other_bookings += 1
# Count by source
covers_by_source[source] += covers
# Count by period (detailed)
if opening_hour_id:
if opening_hour_id not in covers_by_period:
covers_by_period[opening_hour_id] = {
"period_type": period,
"covers": 0,
"bookings": 0
}
covers_by_period[opening_hour_id]["covers"] += covers
covers_by_period[opening_hour_id]["bookings"] += 1
# Count business segments
if booking.is_hotel_guest:
hotel_guest_covers += covers
else:
non_hotel_guest_covers += covers
if booking.is_dbb:
dbb_covers += covers
if booking.is_package:
package_covers += covers
# Track party sizes
if covers > 0:
total_party_sizes.append(covers)
party_sizes_by_period[period].append(covers)
# Build hotel booking numbers mapping
all_booking_numbers, _ = parse_group_exclude_field(
booking.group_exclude_field,
booking.hotel_booking_number
)
if all_booking_numbers:
bookings_with_hotel_link += 1
for hotel_number in all_booking_numbers:
hotel_booking_numbers[hotel_number] = resos_id
# Calculate averages
avg_party_size = sum(total_party_sizes) / len(total_party_sizes) if total_party_sizes else None
avg_by_period = {}
for period, sizes in party_sizes_by_period.items():
avg_by_period[period] = sum(sizes) / len(sizes) if sizes else None
total_covers = breakfast_covers + lunch_covers + afternoon_covers + dinner_covers + other_covers
total_bookings = breakfast_bookings + lunch_bookings + afternoon_bookings + dinner_bookings + other_bookings
distinct_hotel_bookings = len(hotel_booking_numbers)
# Upsert stats
db.execute(
text("""
INSERT INTO resos_bookings_stats (
date,
breakfast_covers, lunch_covers, afternoon_covers, dinner_covers, other_covers, total_covers,
breakfast_bookings, lunch_bookings, afternoon_bookings, dinner_bookings, other_bookings, total_bookings,
covers_by_source, covers_by_period,
hotel_guest_covers, non_hotel_guest_covers, dbb_covers, package_covers,
hotel_booking_numbers, distinct_hotel_bookings, bookings_with_hotel_link,
avg_party_size, avg_party_size_by_period,
aggregated_at
) VALUES (
:date,
:breakfast_covers, :lunch_covers, :afternoon_covers, :dinner_covers, :other_covers, :total_covers,
:breakfast_bookings, :lunch_bookings, :afternoon_bookings, :dinner_bookings, :other_bookings, :total_bookings,
:covers_by_source, :covers_by_period,
:hotel_guest_covers, :non_hotel_guest_covers, :dbb_covers, :package_covers,
:hotel_booking_numbers, :distinct_hotel_bookings, :bookings_with_hotel_link,
:avg_party_size, :avg_party_size_by_period,
NOW()
)
ON CONFLICT (date) DO UPDATE SET
breakfast_covers = :breakfast_covers,
lunch_covers = :lunch_covers,
afternoon_covers = :afternoon_covers,
dinner_covers = :dinner_covers,
other_covers = :other_covers,
total_covers = :total_covers,
breakfast_bookings = :breakfast_bookings,
lunch_bookings = :lunch_bookings,
afternoon_bookings = :afternoon_bookings,
dinner_bookings = :dinner_bookings,
other_bookings = :other_bookings,
total_bookings = :total_bookings,
covers_by_source = :covers_by_source,
covers_by_period = :covers_by_period,
hotel_guest_covers = :hotel_guest_covers,
non_hotel_guest_covers = :non_hotel_guest_covers,
dbb_covers = :dbb_covers,
package_covers = :package_covers,
hotel_booking_numbers = :hotel_booking_numbers,
distinct_hotel_bookings = :distinct_hotel_bookings,
bookings_with_hotel_link = :bookings_with_hotel_link,
avg_party_size = :avg_party_size,
avg_party_size_by_period = :avg_party_size_by_period,
aggregated_at = NOW()
"""),
{
"date": target_date,
"breakfast_covers": breakfast_covers,
"lunch_covers": lunch_covers,
"afternoon_covers": afternoon_covers,
"dinner_covers": dinner_covers,
"other_covers": other_covers,
"total_covers": total_covers,
"breakfast_bookings": breakfast_bookings,
"lunch_bookings": lunch_bookings,
"afternoon_bookings": afternoon_bookings,
"dinner_bookings": dinner_bookings,
"other_bookings": other_bookings,
"total_bookings": total_bookings,
"covers_by_source": json.dumps(dict(covers_by_source)),
"covers_by_period": json.dumps(covers_by_period),
"hotel_guest_covers": hotel_guest_covers,
"non_hotel_guest_covers": non_hotel_guest_covers,
"dbb_covers": dbb_covers,
"package_covers": package_covers,
"hotel_booking_numbers": json.dumps(hotel_booking_numbers),
"distinct_hotel_bookings": distinct_hotel_bookings,
"bookings_with_hotel_link": bookings_with_hotel_link,
"avg_party_size": avg_party_size,
"avg_party_size_by_period": json.dumps(avg_by_period)
}
)
db.commit()
async def update_resos_booking_pace(db):
"""
Update resos_booking_pace table with lead-time snapshots.
Creates 3 rows per date: total, resident, non_resident
"""
logger.info("Updating Resos booking pace table...")
today = date.today()
# Process dates from -30 to +365 (historical + forecast window)
from_date = today - timedelta(days=30)
to_date = today + timedelta(days=365)
current = from_date
while current <= to_date:
# Calculate pace for each type
for pace_type in ['total', 'resident', 'non_resident']:
pace_values = {}
for days_out in PACE_INTERVALS:
snapshot_date = current - timedelta(days=days_out)
# Build query based on pace_type
if pace_type == 'total':
# All valid bookings
result = db.execute(
text("""
SELECT COALESCE(SUM(covers), 0) as total_covers
FROM resos_bookings_data
WHERE booking_date = :target_date
AND status IN :valid_statuses
AND booking_placed <= :snapshot_date
"""),
{
"target_date": current,
"valid_statuses": VALID_STATUSES,
"snapshot_date": snapshot_date
}
)
elif pace_type == 'resident':
# Hotel guests only
result = db.execute(
text("""
SELECT COALESCE(SUM(covers), 0) as total_covers
FROM resos_bookings_data
WHERE booking_date = :target_date
AND status IN :valid_statuses
AND booking_placed <= :snapshot_date
AND is_hotel_guest = true
"""),
{
"target_date": current,
"valid_statuses": VALID_STATUSES,
"snapshot_date": snapshot_date
}
)
else: # non_resident
# Non-hotel guests
result = db.execute(
text("""
SELECT COALESCE(SUM(covers), 0) as total_covers
FROM resos_bookings_data
WHERE booking_date = :target_date
AND status IN :valid_statuses
AND booking_placed <= :snapshot_date
AND (is_hotel_guest = false OR is_hotel_guest IS NULL)
"""),
{
"target_date": current,
"valid_statuses": VALID_STATUSES,
"snapshot_date": snapshot_date
}
)
row = result.fetchone()
pace_values[f"d{days_out}"] = row.total_covers if row else 0
# Upsert pace record
columns = ", ".join(pace_values.keys())
placeholders = ", ".join([f":{k}" for k in pace_values.keys()])
updates = ", ".join([f"{k} = :{k}" for k in pace_values.keys()])
db.execute(
text(f"""
INSERT INTO resos_booking_pace (booking_date, pace_type, {columns}, updated_at)
VALUES (:booking_date, :pace_type, {placeholders}, NOW())
ON CONFLICT (booking_date, pace_type) DO UPDATE SET
{updates},
updated_at = NOW()
"""),
{"booking_date": current, "pace_type": pace_type, **pace_values}
)
current += timedelta(days=1)
db.commit()
logger.info("Resos booking pace table updated (3 types: total, resident, non_resident)")

397
backend/jobs/resos_bookings_sync.py Normal file
View file

@ -0,0 +1,397 @@
"""
Resos Bookings Data Sync Job
Syncs restaurant bookings to resos_bookings_data table
Pattern: Replicates newbook bookings sync but adapted for Resos covers/stats
"""
import json
import logging
import base64
from datetime import date, datetime, timedelta
from typing import Optional, Tuple, List, Dict, Any
from sqlalchemy import text
from database import SyncSessionLocal
from services.resos_client import ResosClient
logger = logging.getLogger(__name__)
# Valid booking statuses for aggregation
# Note: Resos uses 'approved' for confirmed future reservations, 'left' for completed meals
VALID_STATUSES = ('approved', 'arrived', 'seated', 'left')
def get_config_value(db, key: str) -> Optional[str]:
"""Get a configuration value from system_config table."""
result = db.execute(
text("SELECT config_value, is_encrypted FROM system_config WHERE config_key = :key"),
{"key": key}
)
row = result.fetchone()
if not row or not row.config_value:
return None
# Decrypt if encrypted
if row.is_encrypted:
try:
return base64.b64decode(row.config_value.encode()).decode()
except Exception as e:
logger.warning(f"Failed to decrypt {key}: {e}")
return row.config_value
return row.config_value
def load_resos_custom_field_mappings(db) -> Dict[str, Dict[str, Any]]:
"""
Load custom field mappings from resos_custom_field_mapping table.
Returns dict: {field_id: {"maps_to": "hotel_guest", "value_for_true": "Yes"}}
"""
result = db.execute(text("""
SELECT field_id, maps_to, value_for_true
FROM resos_custom_field_mapping
WHERE maps_to != 'ignore'
"""))
mappings = {}
for row in result.fetchall():
mappings[row.field_id] = {
"maps_to": row.maps_to,
"value_for_true": row.value_for_true
}
logger.info(f"Loaded {len(mappings)} custom field mappings")
return mappings
def load_resos_opening_hours_mappings(db) -> Dict[str, Dict[str, str]]:
"""
Load opening hours mappings from resos_opening_hours_mapping table.
Returns dict: {opening_hour_id: {"period_type": "dinner", "display_name": "..."}}
"""
result = db.execute(text("""
SELECT opening_hour_id, period_type, display_name
FROM resos_opening_hours_mapping
WHERE period_type != 'ignore'
"""))
mappings = {}
for row in result.fetchall():
mappings[row.opening_hour_id] = {
"period_type": row.period_type,
"display_name": row.display_name
}
logger.info(f"Loaded {len(mappings)} opening hours mappings")
return mappings
def parse_group_exclude_field(group_exclude_field: Optional[str], primary_booking_number: Optional[str]) -> Tuple[List[str], List[str]]:
"""
Parse group_exclude_field to extract linked bookings and exclude markers.
Args:
group_exclude_field: Raw field like "#12346,#12347,NOT-#56748"
primary_booking_number: Primary booking from hotel_booking_number field
Returns:
(all_booking_numbers, exclude_numbers)
Example:
Input: "#12346,#12347,NOT-#56748", "NB12345"
Returns: (["NB12345", "NB12346", "NB12347"], ["NB56748"])
"""
all_booking_numbers = []
exclude_numbers = []
# Always include primary booking number
if primary_booking_number:
all_booking_numbers.append(primary_booking_number)
if not group_exclude_field:
return all_booking_numbers, exclude_numbers
# Parse comma-separated entries
parts = group_exclude_field.split(',')
for part in parts:
part = part.strip()
if part.upper().startswith('NOT-#'):
# Exclude marker: NOT-#56748 → NB56748
booking_num = part[5:] # Remove "NOT-#"
exclude_numbers.append(f"NB{booking_num}")
elif part.startswith('#'):
# Additional booking: #12346 → NB12346
booking_num = part[1:] # Remove "#"
all_booking_numbers.append(f"NB{booking_num}")
return all_booking_numbers, exclude_numbers
def extract_custom_field_value(custom_fields: List[Dict[str, Any]], field_id: str, cf_mappings: Dict[str, Dict[str, Any]]) -> Optional[Any]:
"""
Extract value from Resos custom fields array.
Args:
custom_fields: Array of custom field objects from Resos API
field_id: The field ID to look for
cf_mappings: Mapping configuration
Returns:
Extracted value (boolean for hotel_guest/dbb/package, string for booking_number, etc.)
"""
if field_id not in cf_mappings:
return None
mapping = cf_mappings[field_id]
maps_to = mapping["maps_to"]
value_for_true = mapping.get("value_for_true")
# Find the field in custom_fields array
field_value = None
field_value_label = None
for cf in custom_fields:
cf_id = cf.get("id") or cf.get("_id") or cf.get("fieldId")
if cf_id == field_id:
field_value = cf.get("value")
field_value_label = cf.get("multipleChoiceValueName") or cf.get("value")
break
if field_value is None and field_value_label is None:
return None
# For boolean mappings (hotel_guest, dbb, package)
if maps_to in ("hotel_guest", "dbb", "package"):
if value_for_true:
return str(field_value_label) == str(value_for_true)
else:
# Auto-detect: check for "yes", "true", "1"
return str(field_value_label).lower() in ("yes", "true", "1")
# For string mappings (booking_number, group_exclude, allergies)
return str(field_value) if field_value else str(field_value_label) if field_value_label else None
async def sync_resos_bookings_data(
from_date: date,
to_date: date,
triggered_by: str = "scheduler"
):
"""
Sync Resos bookings to resos_bookings_data.
Date range: Historical -365 days + Forecast +365 days (daily sync)
PII handling: Remove guest details, store only aggregate covers
"""
logger.info(f"Starting Resos bookings sync from {from_date} to {to_date} (triggered by {triggered_by})")
db = SyncSessionLocal()
try:
# Log sync start
db.execute(
text("""
INSERT INTO sync_log (sync_type, source, started_at, status, date_from, date_to, triggered_by)
VALUES ('bookings_data', 'resos', NOW(), 'running', :from_date, :to_date, :triggered_by)
"""),
{"from_date": from_date, "to_date": to_date, "triggered_by": triggered_by}
)
db.commit()
# Load Resos API key
api_key = get_config_value(db, 'resos_api_key')
if not api_key:
raise Exception("Resos API key not configured")
# Load mappings
cf_mappings = load_resos_custom_field_mappings(db)
oh_mappings = load_resos_opening_hours_mappings(db)
async with ResosClient(api_key=api_key) as client:
# Test connection
if not await client.test_connection():
raise Exception("Resos connection failed")
# Fetch bookings
logger.info(f"Fetching bookings from Resos API for {from_date} to {to_date}")
bookings = await client.get_bookings(from_date, to_date)
logger.info(f"Fetched {len(bookings)} bookings from Resos")
records_created = 0
records_updated = 0
for booking in bookings:
resos_id = booking.get("_id")
if not resos_id:
continue
booking_date_str = booking.get("date")
booking_date_obj = date.fromisoformat(booking_date_str) if booking_date_str else None
if not booking_date_obj:
continue
# Extract opening hour ID and map to period type
opening_hour_id = booking.get("openingHourId")
period_type = None
if opening_hour_id and opening_hour_id in oh_mappings:
period_type = oh_mappings[opening_hour_id]["period_type"]
# Extract custom fields using mappings
custom_fields = booking.get("customFields", [])
is_hotel_guest = None
is_dbb = None
is_package = None
hotel_booking_number = None
group_exclude_field = None
for field_id, mapping in cf_mappings.items():
maps_to = mapping["maps_to"]
if maps_to == "hotel_guest":
is_hotel_guest = extract_custom_field_value(custom_fields, field_id, cf_mappings)
elif maps_to == "dbb":
is_dbb = extract_custom_field_value(custom_fields, field_id, cf_mappings)
elif maps_to == "package":
is_package = extract_custom_field_value(custom_fields, field_id, cf_mappings)
elif maps_to == "booking_number":
hotel_booking_number = extract_custom_field_value(custom_fields, field_id, cf_mappings)
elif maps_to == "group_exclude":
group_exclude_field = extract_custom_field_value(custom_fields, field_id, cf_mappings)
# Remove PII from raw JSON (remove guest object)
raw_booking = {k: v for k, v in booking.items() if k != "guest"}
raw_json_str = json.dumps(raw_booking)
# Extract other booking details
covers = booking.get("people", 0)
status = booking.get("status")
source = booking.get("source")
booking_time_str = booking.get("time")
# Parse table information
tables = booking.get("tables", [])
table_name = tables[0].get("name") if tables else None
table_area = None
if tables and tables[0].get("area"):
table_area = tables[0]["area"].get("name")
# Parse booking placed timestamp
booking_placed_str = booking.get("createdAt")
booking_placed = None
if booking_placed_str:
try:
booking_placed = datetime.fromisoformat(booking_placed_str.replace('Z', '+00:00'))
except:
pass
# Parse notes (sanitized - no PII)
notes_array = booking.get("restaurantNotes", [])
notes = ', '.join([str(note) for note in notes_array]) if notes_array else None
# Check if record exists
existing = db.execute(
text("SELECT id FROM resos_bookings_data WHERE resos_id = :rid"),
{"rid": resos_id}
).fetchone()
# Upsert booking
db.execute(
text("""
INSERT INTO resos_bookings_data (
resos_id, booking_date, booking_time, opening_hour_id, period_type,
covers, status, source, table_name, table_area,
is_hotel_guest, is_dbb, is_package, hotel_booking_number, group_exclude_field,
total_guests, booking_placed, notes, raw_json, fetched_at
) VALUES (
:resos_id, :booking_date, :booking_time, :opening_hour_id, :period_type,
:covers, :status, :source, :table_name, :table_area,
:is_hotel_guest, :is_dbb, :is_package, :hotel_booking_number, :group_exclude_field,
:total_guests, :booking_placed, :notes, :raw_json, NOW()
)
ON CONFLICT (resos_id) DO UPDATE SET
status = :status,
covers = :covers,
period_type = :period_type,
is_hotel_guest = COALESCE(:is_hotel_guest, resos_bookings_data.is_hotel_guest),
is_dbb = COALESCE(:is_dbb, resos_bookings_data.is_dbb),
is_package = COALESCE(:is_package, resos_bookings_data.is_package),
hotel_booking_number = COALESCE(:hotel_booking_number, resos_bookings_data.hotel_booking_number),
group_exclude_field = COALESCE(:group_exclude_field, resos_bookings_data.group_exclude_field),
raw_json = :raw_json,
fetched_at = NOW()
"""),
{
"resos_id": resos_id,
"booking_date": booking_date_obj,
"booking_time": booking_time_str,
"opening_hour_id": opening_hour_id,
"period_type": period_type,
"covers": covers,
"status": status,
"source": source,
"table_name": table_name,
"table_area": table_area,
"is_hotel_guest": is_hotel_guest,
"is_dbb": is_dbb,
"is_package": is_package,
"hotel_booking_number": hotel_booking_number,
"group_exclude_field": group_exclude_field,
"total_guests": covers, # Same as covers for restaurants
"booking_placed": booking_placed,
"notes": notes,
"raw_json": raw_json_str
}
)
if existing:
records_updated += 1
else:
records_created += 1
db.commit()
# Update sync log
db.execute(
text("""
UPDATE sync_log
SET completed_at = NOW(), status = 'success',
records_fetched = :fetched, records_created = :created, records_updated = :updated
WHERE id = (
SELECT id FROM sync_log
WHERE source = 'resos' AND sync_type = 'bookings_data' AND status = 'running'
ORDER BY started_at DESC LIMIT 1
)
"""),
{"fetched": len(bookings), "created": records_created, "updated": records_updated}
)
db.commit()
logger.info(f"Resos bookings sync completed: {records_created} created, {records_updated} updated")
# Trigger aggregation
logger.info("Triggering Resos bookings aggregation...")
try:
import asyncio
from jobs.resos_aggregation import aggregate_resos_bookings
await aggregate_resos_bookings(triggered_by=triggered_by)
logger.info("Resos bookings aggregation completed")
except Exception as agg_error:
logger.warning(f"Resos aggregation failed (non-fatal): {agg_error}")
except Exception as e:
logger.error(f"Resos bookings sync failed: {e}", exc_info=True)
db.execute(
text("""
UPDATE sync_log
SET completed_at = NOW(), status = 'failed', error_message = :error
WHERE id = (
SELECT id FROM sync_log
WHERE source = 'resos' AND sync_type = 'bookings_data' AND status = 'running'
ORDER BY started_at DESC LIMIT 1
)
"""),
{"error": str(e)[:500]}
)
db.commit()
raise
finally:
db.close()

153
backend/jobs/revenue_aggregation.py Normal file
View file

@ -0,0 +1,153 @@
"""
Revenue aggregation - consolidates earned revenue by department
"""
import logging
from datetime import datetime
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
def get_config_value(db, key: str) -> str | None:
"""Get a config value from system_config"""
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = :key"),
{"key": key}
)
row = result.fetchone()
return row.config_value if row else None
def set_config_value(db, key: str, value: str):
"""Set a config value in system_config"""
db.execute(
text("""
INSERT INTO system_config (config_key, config_value, updated_at)
VALUES (:key, :value, NOW())
ON CONFLICT (config_key) DO UPDATE SET
config_value = :value,
updated_at = NOW()
"""),
{"key": key, "value": value}
)
async def aggregate_revenue(since_timestamp: str = None):
"""
Aggregate earned revenue data by department into newbook_net_revenue_data.
- Joins newbook_earned_revenue_data with newbook_gl_accounts to get department
- Sums net amounts by date and department
- Only processes dates with data fetched since last aggregation (or all if first run)
Args:
since_timestamp: Optional timestamp to process data from (ISO format)
If not provided, uses last_revenue_aggregation_at config
"""
logger.info("Starting revenue aggregation...")
db = SyncSessionLocal()
try:
# Get last aggregation time if not provided
if since_timestamp is None:
since_timestamp = get_config_value(db, 'last_revenue_aggregation_at')
# Find dates with new/updated data
if since_timestamp:
logger.info(f"Aggregating revenue data updated since {since_timestamp}")
result = db.execute(
text("""
SELECT DISTINCT date
FROM newbook_earned_revenue_data
WHERE fetched_at > :since
ORDER BY date
"""),
{"since": since_timestamp}
)
else:
logger.info("Aggregating all revenue data (first run)")
result = db.execute(
text("""
SELECT DISTINCT date
FROM newbook_earned_revenue_data
ORDER BY date
""")
)
dates_to_process = [row.date for row in result.fetchall()]
if not dates_to_process:
logger.info("No new revenue data to aggregate")
return {"dates_processed": 0, "message": "No new data"}
logger.info(f"Found {len(dates_to_process)} dates to aggregate")
# Aggregate each date
for target_date in dates_to_process:
# Get totals by department for this date
result = db.execute(
text("""
SELECT
COALESCE(g.department, 'other') as department,
SUM(e.amount_net) as total_net
FROM newbook_earned_revenue_data e
LEFT JOIN newbook_gl_accounts g ON e.gl_code = g.gl_code
WHERE e.date = :date
GROUP BY g.department
"""),
{"date": target_date}
)
totals = {"accommodation": 0, "dry": 0, "wet": 0}
for row in result.fetchall():
if row.department in totals:
totals[row.department] = float(row.total_net or 0)
# Upsert into newbook_net_revenue_data
db.execute(
text("""
INSERT INTO newbook_net_revenue_data (date, accommodation, dry, wet, aggregated_at)
VALUES (:date, :accommodation, :dry, :wet, NOW())
ON CONFLICT (date) DO UPDATE SET
accommodation = :accommodation,
dry = :dry,
wet = :wet,
aggregated_at = NOW()
"""),
{
"date": target_date,
"accommodation": round(totals["accommodation"], 2),
"dry": round(totals["dry"], 2),
"wet": round(totals["wet"], 2)
}
)
db.commit()
# Update last aggregation timestamp
set_config_value(db, 'last_revenue_aggregation_at', datetime.now().isoformat())
db.commit()
logger.info(f"Revenue aggregation complete: {len(dates_to_process)} dates")
return {
"dates_processed": len(dates_to_process),
"message": f"Aggregated {len(dates_to_process)} dates"
}
except Exception as e:
logger.error(f"Revenue aggregation failed: {e}")
db.rollback()
raise
finally:
db.close()
async def backfill_revenue_aggregation():
"""
Backfill all historical revenue data.
Forces re-aggregation of all dates regardless of last run time.
"""
logger.info("Starting revenue backfill aggregation...")
# Pass epoch time to force processing all data
return await aggregate_revenue(since_timestamp="1970-01-01T00:00:00")

165
backend/jobs/scrape_booking_rates.py Normal file
View file

@ -0,0 +1,165 @@
"""
Scheduled Booking.com Rate Scraping Job
Priority-based scheduling for 365-day coverage (all queued daily):
- High (priority 10): next 30 days
- Medium (priority 5): days 31-180
- Low (priority 2): days 181-365
Queue processes in priority order. If rate-limited/blocked, lower priority
dates remain queued for the next run.
Uses a queue-based approach:
1. Populate the queue with dates and priorities
2. Process the queue in priority order
3. Failed dates are retried (up to 3 attempts)
4. On blocking, the queue pauses and resumes after cooldown
Schedule: Daily at configurable time (default 05:30)
"""
import asyncio
import logging
from datetime import date, timedelta
from sqlalchemy import text
from database import SyncSessionLocal
from services.booking_scraper import (
populate_queue,
process_queue,
clear_old_queue_items,
cleanup_stale_batches,
get_scrape_config,
)
logger = logging.getLogger(__name__)
# Priority levels (higher = processed first)
PRIORITY_HIGH = 10 # 0-30 days
PRIORITY_MEDIUM = 5 # 31-180 days
PRIORITY_LOW = 2 # 181-365 days
def get_high_priority_dates() -> list[date]:
"""High priority: today + 30 days."""
today = date.today()
return [today + timedelta(days=i) for i in range(31)]
def get_medium_priority_dates() -> list[date]:
"""Medium priority: days 31-180."""
today = date.today()
return [today + timedelta(days=i) for i in range(31, 181)]
def get_low_priority_dates() -> list[date]:
"""Low priority: days 181-365."""
today = date.today()
return [today + timedelta(days=i) for i in range(181, 366)]
def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
"""
For a target date, determine its priority tier and when it will next be scraped.
Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'.
All dates are queued daily, so next scrape is always today (or tomorrow if
today's run has passed).
"""
today = date.today()
offset = (target_date - today).days
if offset < 0:
return ('none', None)
if offset > 365:
return ('none', None)
# All tiers run daily - next scrape is today
if offset <= 30:
return ('high', today)
elif offset <= 180:
return ('medium', today)
else:
return ('low', today)
def run_scheduled_booking_scrape():
"""
Main scheduled job: populate queue with today's dates, then process.
"""
db = SyncSessionLocal()
try:
# Check if scraper is enabled
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
).fetchone()
if not result or result.config_value != 'true':
logger.debug("Scheduled booking scrape skipped (disabled)")
return
if not get_scrape_config(db):
logger.warning("Scheduled booking scrape skipped (no location configured)")
return
# Clean up stale running batches and old queue items
cleanup_stale_batches(db, max_age_minutes=120)
clear_old_queue_items(db, days=3)
# Gather dates with priorities
high = get_high_priority_dates()
medium = get_medium_priority_dates()
low = get_low_priority_dates()
priorities = {}
for d in high:
priorities[d] = PRIORITY_HIGH
for d in medium:
priorities[d] = max(priorities.get(d, 0), PRIORITY_MEDIUM)
for d in low:
priorities[d] = max(priorities.get(d, 0), PRIORITY_LOW)
all_dates = sorted(priorities.keys())
if not all_dates:
logger.info("Scheduled booking scrape: no dates to scrape today")
return
logger.info(
f"Scheduled booking scrape: queuing {len(all_dates)} dates "
f"(high={len(high)}, medium={len(medium)}, low={len(low)})"
)
# Populate queue
populate_queue(db, all_dates, priorities)
# Process queue
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
result = loop.run_until_complete(process_queue(db))
if result.get('success'):
logger.info(
f"Scheduled booking scrape completed: "
f"{result.get('dates_completed', 0)} dates, "
f"{result.get('rates_scraped', 0)} rates"
)
elif result.get('blocked'):
logger.warning(
f"Scheduled booking scrape blocked: {result.get('block_reason')}. "
f"Completed {result.get('dates_completed', 0)} dates. "
f"Remaining dates stay queued for retry."
)
else:
logger.error(f"Scheduled booking scrape failed: {result.get('error')}")
finally:
loop.close()
except Exception as e:
logger.error(f"Scheduled booking scrape error: {e}", exc_info=True)
finally:
db.close()
async def run_scheduled_booking_scrape_async():
"""Async wrapper for APScheduler."""
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, run_scheduled_booking_scrape)

197
backend/jobs/weekly_forecast_snapshot.py Normal file
View file

@ -0,0 +1,197 @@
"""
Weekly Forecast Snapshot Job
Automatically creates blended forecast snapshots using MAPE-weighted model blending with 60/40 budget/prior year blend.
Uses the blended_tuned_weighted service for accuracy-optimized forecasts.
"""
import logging
import uuid
from datetime import date, datetime, timedelta
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
def get_config_value(db, key: str, default: str = None) -> str:
"""Get a config value from system_config"""
try:
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = :key"),
{"key": key}
)
row = result.fetchone()
if row and row.config_value:
return row.config_value
return default
except Exception as e:
logger.error(f"Error getting config {key}: {e}")
return default
def is_forecast_snapshot_enabled(db) -> bool:
"""Check if automated forecast snapshots are enabled"""
value = get_config_value(db, "forecast_snapshot_enabled")
if value:
return value.lower() in ('true', '1', 'yes', 'enabled')
return False
def get_forecast_snapshot_days_ahead(db) -> int:
"""Get number of days ahead to forecast"""
days_str = get_config_value(db, "forecast_snapshot_days_ahead", "90")
try:
return int(days_str)
except ValueError:
return 90
async def run_weekly_forecast_snapshot():
"""
Run weekly blended forecast snapshot using MAPE-weighted + 60/40 blend.
This is the single source of truth for forecast snapshots.
- Stage 1: MAPE-weighted blend of Prophet, XGBoost, CatBoost (+ Pickup for pace metrics)
- Stage 2: 60% model blend + 40% budget (revenue) or prior year (non-revenue)
"""
db = SyncSessionLocal()
run_id = str(uuid.uuid4())
try:
# Check if enabled
if not is_forecast_snapshot_enabled(db):
logger.info("Weekly forecast snapshot is disabled, skipping")
return
days_ahead = get_forecast_snapshot_days_ahead(db)
forecast_from = date.today()
forecast_to = date.today() + timedelta(days=days_ahead)
logger.info(f"Starting weekly blended forecast snapshot: {forecast_from} to {forecast_to}")
# Log run start
db.execute(
text("""
INSERT INTO forecast_runs (
run_id, run_type, started_at, status,
forecast_from, forecast_to, models_run, triggered_by
) VALUES (
:run_id, 'scheduled', NOW(), 'running',
:forecast_from, :forecast_to, :models, :triggered_by
)
"""),
{
"run_id": run_id,
"forecast_from": forecast_from,
"forecast_to": forecast_to,
"models": '["blended"]',
"triggered_by": "forecast_snapshot"
}
)
db.commit()
# Get active metrics
result = db.execute(
text("""
SELECT metric_code, metric_name
FROM forecast_metrics
WHERE is_active = TRUE
""")
)
metrics = result.fetchall()
# Import MAPE-weighted blended model with 60/40 budget blend
from services.forecasting.blended_tuned_weighted import run_blended_tuned_weighted_forecast
# Run blended forecast for each metric
total_forecasts = 0
for metric in metrics:
metric_code = metric.metric_code
try:
logger.info(f"Generating MAPE-weighted + 60/40 blended forecast for {metric_code}")
forecasts = await run_blended_tuned_weighted_forecast(
db=db,
metric_code=metric_code,
start_date=forecast_from,
end_date=forecast_to,
save_to_db=True,
run_id=run_id
# apply_60_40_blend defaults to True
)
total_forecasts += len(forecasts)
logger.info(f"Generated {len(forecasts)} MAPE-weighted + 60/40 forecasts for {metric_code}")
except Exception as e:
logger.error(f"Blended forecast failed for {metric_code}: {e}")
db.rollback()
continue
# Update run status to success
db.execute(
text("""
UPDATE forecast_runs
SET completed_at = NOW(), status = 'success'
WHERE run_id = :run_id
"""),
{"run_id": run_id}
)
db.commit()
# Log completion to sync_log
db.execute(
text("""
INSERT INTO sync_log (sync_type, source, started_at, completed_at, status, records_created, triggered_by)
VALUES (:sync_type, :source, :started_at, :completed_at, :status, :records_created, :triggered_by)
"""),
{
"sync_type": "forecast_snapshot",
"source": "blended_tuned_weighted",
"started_at": datetime.now(),
"completed_at": datetime.now(),
"status": "success",
"records_created": total_forecasts,
"triggered_by": "scheduler"
}
)
db.commit()
logger.info(f"Weekly forecast snapshot completed: {total_forecasts} total forecasts generated")
except Exception as e:
logger.error(f"Weekly forecast snapshot failed: {e}")
# Rollback and update run status
db.rollback()
try:
db.execute(
text("""
UPDATE forecast_runs
SET completed_at = NOW(), status = 'failed', error_message = :error
WHERE run_id = :run_id
"""),
{"run_id": run_id, "error": str(e)}
)
db.commit()
except:
pass
# Log error to sync_log
try:
db.execute(
text("""
INSERT INTO sync_log (sync_type, source, started_at, completed_at, status, error_message, triggered_by)
VALUES (:sync_type, :source, :started_at, :completed_at, :status, :error_message, :triggered_by)
"""),
{
"sync_type": "forecast_snapshot",
"source": "blended_tuned_weighted",
"started_at": datetime.now(),
"completed_at": datetime.now(),
"status": "error",
"error_message": str(e),
"triggered_by": "scheduler"
}
)
db.commit()
except:
pass
raise
finally:
db.close()