From d5ce38859d1181b487ab1db0405ca3afa8a8d497 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Tue, 14 Jul 2026 09:39:35 +0000 Subject: [PATCH] =?UTF-8?q?Fix=20booking=20scrape=20date=20selection=20?= =?UTF-8?q?=E2=80=94=20oldest-first=20tiered=20distribution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Medium and low priority tiers were generating full static date ranges (150 and 185 dates) every day. With a queue limit of 200, high (31) + medium (150) consumed the entire budget, leaving only ~19 slots for low priority — causing the observed ~6 month cap. Medium now selects the 60 oldest-scraped (or never-scraped) dates from the days 31-180 window; low selects the 30 oldest from days 181-365. Per-run budget drops from ~365 to ~121 dates, and coverage naturally cycles through the full year: medium every ~2-3 days, low every ~6-7 days. Co-Authored-By: Claude Sonnet 4.6 --- backend/jobs/scrape_booking_rates.py | 95 +++++++++++++++++++++------- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/backend/jobs/scrape_booking_rates.py b/backend/jobs/scrape_booking_rates.py index e9371c5..1b74a25 100644 --- a/backend/jobs/scrape_booking_rates.py +++ b/backend/jobs/scrape_booking_rates.py @@ -1,16 +1,23 @@ """ Scheduled Booking.com Rate Scraping Job -Priority-based scheduling for 365-day coverage (all queued daily): -- High (priority 10): next 30 days -- Medium (priority 5): days 31-180 -- Low (priority 2): days 181-365 +Tiered scheduling for 365-day coverage: +- High (priority 10): next 30 days — all 31 dates scraped every day +- Medium (priority 5): days 31-180 — up to 60 oldest-scraped dates per day +- Low (priority 2): days 181-365 — up to 30 oldest-scraped dates per day -Queue processes in priority order. If rate-limited/blocked, lower priority -dates remain queued for the next run. +Per-run budget: ~121 dates (30 high + 60 medium + 30 low). +Medium and low tiers select whichever dates in their window have the oldest +(or missing) scrape data first, naturally distributing coverage across the +full year without hammering all 365 dates every day. + +Approximate refresh cadence: +- High: daily +- Medium (150 dates / 60 per day): every ~2-3 days +- Low (185 dates / 30 per day): every ~6 days Uses a queue-based approach: -1. Populate the queue with dates and priorities +1. Populate the queue with the selected dates and priorities 2. Process the queue in priority order 3. Failed dates are retried (up to 3 attempts) 4. On blocking, the queue pauses and resumes after cooldown @@ -22,6 +29,7 @@ import logging from datetime import date, timedelta from sqlalchemy import text +from sqlalchemy.orm import Session from database import SyncSessionLocal from services.booking_scraper import ( populate_queue, @@ -38,32 +46,69 @@ PRIORITY_HIGH = 10 # 0-30 days PRIORITY_MEDIUM = 5 # 31-180 days PRIORITY_LOW = 2 # 181-365 days +# Per-run date budget for medium and low tiers +MEDIUM_DATES_PER_RUN = 60 # out of ~150 in window +LOW_DATES_PER_RUN = 30 # out of ~185 in window + def get_high_priority_dates() -> list[date]: - """High priority: today + 30 days.""" + """High priority: all of today through +30 days, scraped every run.""" today = date.today() return [today + timedelta(days=i) for i in range(31)] -def get_medium_priority_dates() -> list[date]: - """Medium priority: days 31-180.""" - today = date.today() - return [today + timedelta(days=i) for i in range(31, 181)] +def _get_oldest_scraped_dates(db: Session, start: date, end: date, limit: int) -> list[date]: + """ + From the date window [start, end], return up to `limit` dates ordered by + oldest last-scrape first (never-scraped dates come first via NULLS FIRST). + + Joining against booking_com_rates means we naturally pick whichever dates + in the window are most stale or have no data yet. + """ + rows = db.execute( + text(""" + SELECT s.rate_date::date AS rate_date, + MAX(r.scraped_at) AS last_scraped + FROM generate_series(:start::date, :end::date, '1 day'::interval) AS s(rate_date) + LEFT JOIN booking_com_rates r ON r.rate_date = s.rate_date::date + GROUP BY s.rate_date + ORDER BY last_scraped ASC NULLS FIRST + LIMIT :limit + """), + {'start': start, 'end': end, 'limit': limit} + ).fetchall() + return [row.rate_date for row in rows] -def get_low_priority_dates() -> list[date]: - """Low priority: days 181-365.""" +def get_medium_priority_dates(db: Session, limit: int = MEDIUM_DATES_PER_RUN) -> list[date]: + """Medium priority: up to `limit` oldest-scraped dates in the days 31-180 window.""" today = date.today() - return [today + timedelta(days=i) for i in range(181, 366)] + return _get_oldest_scraped_dates( + db, + start=today + timedelta(days=31), + end=today + timedelta(days=180), + limit=limit, + ) + + +def get_low_priority_dates(db: Session, limit: int = LOW_DATES_PER_RUN) -> list[date]: + """Low priority: up to `limit` oldest-scraped dates in the days 181-365 window.""" + today = date.today() + return _get_oldest_scraped_dates( + db, + start=today + timedelta(days=181), + end=today + timedelta(days=365), + limit=limit, + ) def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]: """ - For a target date, determine its priority tier and when it will next be scraped. + For a target date, determine its priority tier and approximate next scrape date. Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'. - All dates are queued daily, so next scrape is always today (or tomorrow if - today's run has passed). + High dates are scraped every run; medium/low estimates reflect typical cadence + based on the per-run budget vs window size. """ today = date.today() offset = (target_date - today).days @@ -73,13 +118,15 @@ def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]: if offset > 365: return ('none', None) - # All tiers run daily - next scrape is today if offset <= 30: + # Scraped every day return ('high', today) elif offset <= 180: - return ('medium', today) + # ~150 dates, 60/day → ~2-3 day cadence + return ('medium', today + timedelta(days=3)) else: - return ('low', today) + # ~185 dates, 30/day → ~6-7 day cadence + return ('low', today + timedelta(days=7)) def run_scheduled_booking_scrape(): @@ -104,10 +151,10 @@ def run_scheduled_booking_scrape(): cleanup_stale_batches(db, max_age_minutes=120) clear_old_queue_items(db, days=3) - # Gather dates with priorities + # Gather dates with priorities — medium/low select oldest-scraped first high = get_high_priority_dates() - medium = get_medium_priority_dates() - low = get_low_priority_dates() + medium = get_medium_priority_dates(db) + low = get_low_priority_dates(db) priorities = {} for d in high: