hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add Rate Monitor app — Booking.com + direct booking engine competitor rates

Browse source 
Combines Booking.com Playwright scraper (from forecasting), direct booking
engine scraper (ported from laptop-archive/guestline-monitor), and Newbook
own-hotel rates into one focused tool. Four views: Bookability, Market View
(with price index badges + direct rate sub-rows), Direct Rates (per-competitor
room breakdown, min-stay flags, hotel config/discovery), Rate Analysis
(advance purchase curve, DOW chart, rate timeline, comparison table).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 12:06:30 +00:00
commit e05054172f
50 changed files with 11860 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
backend/jobs/__init__.py Normal file
View file

370
backend/jobs/fetch_current_rates.py Normal file
View file

@ -0,0 +1,370 @@
"""
Fetch Current Rates Job
Fetches current rack rates from Newbook API and populates newbook_current_rates table.
These rates are used by pickup-v2 model for upper bound calculations in confidence shading.
Uses a snapshot model - only inserts new rows when rates change, otherwise updates last_verified_at.
This allows tracking rate history over time.
Schedule: Daily at 5:20 AM (before pace snapshot runs)
Processing: Day-by-day with progressive DB commits. Each date is fully processed
(single-night fetch + inline multi-night verification) and saved before moving to the next.
If the job fails partway, all previously processed dates are preserved.
"""
import json
import logging
import re
from datetime import date, timedelta
from decimal import Decimal
from typing import Dict, Any, Optional, Set
import asyncio
from sqlalchemy import text
from database import SyncSessionLocal
logger = logging.getLogger(__name__)
COMMIT_BATCH_SIZE = 10 # Commit to DB every N days
def rates_changed(old_rate: Optional[Dict], new_rate: Dict) -> bool:
"""
Compare old and new rates to determine if they've changed.
Compares gross rate, net rate, and tariff availability status.
Returns True if rates have changed, False if they're the same.
"""
if old_rate is None:
return True # No existing rate, need to insert
# Compare gross and net rates
old_gross = float(old_rate.get('rate_gross') or 0)
new_gross = float(new_rate.get('gross_rate') or 0)
if abs(old_gross - new_gross) > 0.01:
return True
old_net = float(old_rate.get('rate_net') or 0)
new_net = float(new_rate.get('net_rate') or 0)
if abs(old_net - new_net) > 0.01:
return True
# Compare tariff availability
old_tariffs = old_rate.get('tariffs_data', {})
if isinstance(old_tariffs, str):
try:
old_tariffs = json.loads(old_tariffs)
except json.JSONDecodeError:
old_tariffs = {}
new_tariffs = new_rate.get('tariffs_data', {})
old_tariff_list = old_tariffs.get('tariffs', [])
new_tariff_list = new_tariffs.get('tariffs', [])
# Different number of tariffs
if len(old_tariff_list) != len(new_tariff_list):
return True
# Compare each tariff's key attributes
for old_t, new_t in zip(old_tariff_list, new_tariff_list):
# Name changed
if old_t.get('name') != new_t.get('name'):
return True
# Availability status changed
if old_t.get('success') != new_t.get('success'):
return True
# Rate changed significantly
old_rate_val = float(old_t.get('rate') or 0)
new_rate_val = float(new_t.get('rate') or 0)
if abs(old_rate_val - new_rate_val) > 0.01:
return True
# Min stay changed
if old_t.get('min_stay') != new_t.get('min_stay'):
return True
# Multi-night availability changed
if old_t.get('available_for_min_stay') != new_t.get('available_for_min_stay'):
return True
return False
def save_rate_snapshot(db, category_id: str, rate_date: date, rate: Dict) -> str:
"""
Save rate to database using snapshot logic.
If rate has changed from latest version, insert new row.
If rate is the same, just update last_verified_at.
Returns: 'inserted', 'verified', or 'error'
"""
gross_rate = rate.get('gross_rate')
net_rate = rate.get('net_rate')
tariffs_data = rate.get('tariffs_data', {})
# Get the latest rate for this category/date
existing = db.execute(
text("""
SELECT id, rate_gross, rate_net, tariffs_data
FROM newbook_current_rates
WHERE category_id = :category_id AND rate_date = :rate_date
ORDER BY valid_from DESC
LIMIT 1
"""),
{"category_id": category_id, "rate_date": rate_date}
).fetchone()
if existing:
existing_dict = {
'rate_gross': existing.rate_gross,
'rate_net': existing.rate_net,
'tariffs_data': existing.tariffs_data
}
else:
existing_dict = None
if rates_changed(existing_dict, rate):
# Rates changed - insert new snapshot
db.execute(
text("""
INSERT INTO newbook_current_rates
(category_id, rate_date, rate_gross, rate_net, tariffs_data, valid_from, last_verified_at)
VALUES (:category_id, :rate_date, :rate_gross, :rate_net,
CAST(:tariffs_data AS jsonb), NOW(), NOW())
"""),
{
"category_id": category_id,
"rate_date": rate_date,
"rate_gross": gross_rate,
"rate_net": net_rate,
"tariffs_data": json.dumps(tariffs_data)
}
)
return 'inserted'
else:
# Rates unchanged - just verify
db.execute(
text("""
UPDATE newbook_current_rates
SET last_verified_at = NOW()
WHERE id = :id
"""),
{"id": existing.id}
)
return 'verified'
def needs_multi_night_check(tariff: Dict, days_ahead: int) -> Optional[int]:
"""
Check if a tariff needs multi-night verification.
Returns the min_stay value if a multi-night check is needed, None otherwise.
Skips tariffs with advance booking restrictions that aren't met.
"""
min_stay = tariff.get('min_stay')
if not min_stay or min_stay <= 1:
return None
if tariff.get('success', False):
return None # Already available as single-night, no recheck needed
# Check for advance booking requirement
message = tariff.get('message', '') or ''
advance_match = re.search(r'(\d+)\s*days?\s*in\s*advance', message, re.IGNORECASE)
if advance_match:
min_advance_days = int(advance_match.group(1))
if days_ahead < min_advance_days:
return None # Within advance period - recheck won't help
return min_stay
async def run_fetch_current_rates(horizon_days: int = 720, start_date: date = None):
"""
Fetch current rates for all included categories and store in database.
Args:
horizon_days: Number of days ahead to fetch (default 720 for scheduled, configurable for manual)
start_date: Start date for fetch (default today)
Processing: Day-by-day with progressive commits.
For each date:
1. Fetch single-night rates (all categories in one API call)
2. Check if any tariffs need multi-night verification
3. If so, run multi-night check immediately for that date
4. Save all rates for that date to DB
5. Commit every COMMIT_BATCH_SIZE days
This means if the job fails at day 400, the first 390+ days are already saved.
"""
logger.info(f"Starting current rates fetch ({horizon_days} days)")
db = next(iter([SyncSessionLocal()]))
today = start_date or date.today()
try:
# Get VAT rate from config
vat_result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'accommodation_vat_rate'")
).fetchone()
vat_rate_str = vat_result.config_value if vat_result and vat_result.config_value else '0.20'
# Get all included room categories
cat_result = db.execute(
text("SELECT site_id FROM newbook_room_categories WHERE is_included = true")
)
included_categories = set(row.site_id for row in cat_result.fetchall())
if not included_categories:
logger.warning("No included room categories found")
return
logger.info(f"Fetching rates for {len(included_categories)} categories")
# Import rates client
import base64
from services.newbook_rates_client import NewbookRatesClient
# Credentials: central Settings service first, app-local config fallback
from services.central_settings import get_newbook_credentials_sync
creds = get_newbook_credentials_sync()
if not creds:
config_result = db.execute(
text("""
SELECT config_key, config_value, COALESCE(is_encrypted, false) as is_encrypted
FROM system_config
WHERE config_key IN ('newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region')
""")
)
config = {}
for row in config_result.fetchall():
value = row.config_value
if row.is_encrypted and value:
try:
value = base64.b64decode(value.encode()).decode()
except Exception:
pass # Use raw value if decryption fails
config[row.config_key] = value
if not all(k in config for k in ['newbook_api_key', 'newbook_username', 'newbook_password', 'newbook_region']):
logger.error("Newbook credentials not configured")
return
creds = {
'api_key': config['newbook_api_key'],
'username': config['newbook_username'],
'password': config['newbook_password'],
'region': config['newbook_region'],
}
# Create client
client = NewbookRatesClient(
api_key=creds['api_key'],
username=creds['username'],
password=creds['password'],
region=creds['region'],
vat_rate=Decimal(vat_rate_str)
)
async with client:
inserted_total = 0
verified_total = 0
multi_night_checks = 0
skipped_advance = 0
current_date = today
day_count = 0
while current_date <= today + timedelta(days=horizon_days):
day_count += 1
days_ahead = (current_date - today).days
try:
# Step 1: Fetch single-night rates for all categories on this date
day_rates = await client.fetch_single_date_all_categories(
current_date, guests_adults=2, guests_children=0
)
# Step 2: Check for multi-night verification needs and run inline
# Collect unique min_stay values needed for this date
nights_needed: Set[int] = set()
for cat_id, rates in day_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
for tariff in rate.get('tariffs_data', {}).get('tariffs', []):
check = needs_multi_night_check(tariff, days_ahead)
if check:
nights_needed.add(check)
elif tariff.get('min_stay') and tariff['min_stay'] > 1 and not tariff.get('success', False):
skipped_advance += 1
# Step 3: Run multi-night checks for this date if needed
multi_night_results: Dict[int, Dict[str, Dict[str, bool]]] = {}
for nights in sorted(nights_needed):
try:
result = await client.fetch_multi_night_for_date(
current_date, nights
)
multi_night_results[nights] = result
multi_night_checks += 1
await asyncio.sleep(1.0) # Rate limiting
except Exception as e:
logger.warning(f"Multi-night check failed for {current_date} ({nights}n): {e}")
# Step 4: Update tariffs with multi-night results and save to DB
for cat_id, rates in day_rates.items():
if cat_id not in included_categories:
continue
for rate in rates:
tariffs_data = rate.get('tariffs_data', {})
# Apply multi-night results to tariffs
for tariff in tariffs_data.get('tariffs', []):
min_stay = tariff.get('min_stay')
if min_stay and min_stay > 1 and min_stay in multi_night_results:
cat_availability = multi_night_results[min_stay].get(cat_id, {})
tariff_name = tariff.get('name', '')
tariff['available_for_min_stay'] = cat_availability.get(tariff_name, False)
# Save to DB
result = save_rate_snapshot(db, cat_id, current_date, rate)
if result == 'inserted':
inserted_total += 1
elif result == 'verified':
verified_total += 1
if day_count % 50 == 0 or nights_needed:
logger.info(
f"Day {day_count}/{horizon_days}: {current_date}"
f" | {inserted_total} new, {verified_total} verified"
f"{f' | {len(nights_needed)} multi-night checks' if nights_needed else ''}"
)
except Exception as e:
logger.warning(f"Failed to fetch rates for {current_date}: {e}")
# Step 5: Commit periodically
if day_count % COMMIT_BATCH_SIZE == 0:
db.commit()
current_date += timedelta(days=1)
await asyncio.sleep(1.0) # Rate limiting between days
# Final commit for remaining days
db.commit()
if skipped_advance > 0:
logger.info(f"Skipped {skipped_advance} multi-night checks (advance booking restriction)")
logger.info(
f"Complete: {inserted_total} new snapshots, {verified_total} verified unchanged, "
f"{multi_night_checks} multi-night checks"
)
logger.info("Current rates fetch completed successfully")
except Exception as e:
logger.error(f"Current rates fetch failed: {e}")
db.rollback()
raise
finally:
db.close()

165
backend/jobs/scrape_booking_rates.py Normal file
View file

@ -0,0 +1,165 @@
"""
Scheduled Booking.com Rate Scraping Job
Priority-based scheduling for 365-day coverage (all queued daily):
- High (priority 10): next 30 days
- Medium (priority 5): days 31-180
- Low (priority 2): days 181-365
Queue processes in priority order. If rate-limited/blocked, lower priority
dates remain queued for the next run.
Uses a queue-based approach:
1. Populate the queue with dates and priorities
2. Process the queue in priority order
3. Failed dates are retried (up to 3 attempts)
4. On blocking, the queue pauses and resumes after cooldown
Schedule: Daily at configurable time (default 05:30)
"""
import asyncio
import logging
from datetime import date, timedelta
from sqlalchemy import text
from database import SyncSessionLocal
from services.booking_scraper import (
populate_queue,
process_queue,
clear_old_queue_items,
cleanup_stale_batches,
get_scrape_config,
)
logger = logging.getLogger(__name__)
# Priority levels (higher = processed first)
PRIORITY_HIGH = 10 # 0-30 days
PRIORITY_MEDIUM = 5 # 31-180 days
PRIORITY_LOW = 2 # 181-365 days
def get_high_priority_dates() -> list[date]:
"""High priority: today + 30 days."""
today = date.today()
return [today + timedelta(days=i) for i in range(31)]
def get_medium_priority_dates() -> list[date]:
"""Medium priority: days 31-180."""
today = date.today()
return [today + timedelta(days=i) for i in range(31, 181)]
def get_low_priority_dates() -> list[date]:
"""Low priority: days 181-365."""
today = date.today()
return [today + timedelta(days=i) for i in range(181, 366)]
def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
"""
For a target date, determine its priority tier and when it will next be scraped.
Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'.
All dates are queued daily, so next scrape is always today (or tomorrow if
today's run has passed).
"""
today = date.today()
offset = (target_date - today).days
if offset < 0:
return ('none', None)
if offset > 365:
return ('none', None)
# All tiers run daily - next scrape is today
if offset <= 30:
return ('high', today)
elif offset <= 180:
return ('medium', today)
else:
return ('low', today)
def run_scheduled_booking_scrape():
"""
Main scheduled job: populate queue with today's dates, then process.
"""
db = SyncSessionLocal()
try:
# Check if scraper is enabled
result = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
).fetchone()
if not result or result.config_value != 'true':
logger.debug("Scheduled booking scrape skipped (disabled)")
return
if not get_scrape_config(db):
logger.warning("Scheduled booking scrape skipped (no location configured)")
return
# Clean up stale running batches and old queue items
cleanup_stale_batches(db, max_age_minutes=120)
clear_old_queue_items(db, days=3)
# Gather dates with priorities
high = get_high_priority_dates()
medium = get_medium_priority_dates()
low = get_low_priority_dates()
priorities = {}
for d in high:
priorities[d] = PRIORITY_HIGH
for d in medium:
priorities[d] = max(priorities.get(d, 0), PRIORITY_MEDIUM)
for d in low:
priorities[d] = max(priorities.get(d, 0), PRIORITY_LOW)
all_dates = sorted(priorities.keys())
if not all_dates:
logger.info("Scheduled booking scrape: no dates to scrape today")
return
logger.info(
f"Scheduled booking scrape: queuing {len(all_dates)} dates "
f"(high={len(high)}, medium={len(medium)}, low={len(low)})"
)
# Populate queue
populate_queue(db, all_dates, priorities)
# Process queue
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
result = loop.run_until_complete(process_queue(db))
if result.get('success'):
logger.info(
f"Scheduled booking scrape completed: "
f"{result.get('dates_completed', 0)} dates, "
f"{result.get('rates_scraped', 0)} rates"
)
elif result.get('blocked'):
logger.warning(
f"Scheduled booking scrape blocked: {result.get('block_reason')}. "
f"Completed {result.get('dates_completed', 0)} dates. "
f"Remaining dates stay queued for retry."
)
else:
logger.error(f"Scheduled booking scrape failed: {result.get('error')}")
finally:
loop.close()
except Exception as e:
logger.error(f"Scheduled booking scrape error: {e}", exc_info=True)
finally:
db.close()
async def run_scheduled_booking_scrape_async():
"""Async wrapper for APScheduler."""
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, run_scheduled_booking_scrape)

43
backend/jobs/scrape_direct_rates.py Normal file
View file

@ -0,0 +1,43 @@
"""
Daily job: scrape all enabled direct competitor hotels.
Hotels are scraped sequentially — the scraper enforces 10s delays per date
to avoid rate-limiting, so concurrent scraping is not beneficial.
"""
import logging
from sqlalchemy import text
from database import SyncSessionLocal
from services.direct_scraper import run_scrape
log = logging.getLogger(__name__)
def run_scrape_all_direct():
db = SyncSessionLocal()
try:
rows = db.execute(
text("""SELECT id, name, profile_name, params
FROM direct_competitor_hotels
WHERE scrape_enabled = true
ORDER BY id""")
).mappings().fetchall()
finally:
db.close()
if not rows:
log.info("No direct competitor hotels enabled for scraping")
return
log.info(f"Starting direct rate scrape for {len(rows)} hotels")
for hotel in rows:
try:
log.info(f"Scraping {hotel['name']} ({hotel['profile_name']})")
run_scrape(
hotel_id=hotel["id"],
profile_name=hotel["profile_name"],
params=hotel["params"],
)
except Exception as e:
log.error(f"Direct scrape failed for hotel {hotel['id']} ({hotel['name']}): {e}")
log.info("Direct rate scrape complete")