Fix booking scrape date selection — oldest-first tiered distribution

Medium and low priority tiers were generating full static date ranges
(150 and 185 dates) every day. With a queue limit of 200, high (31) +
medium (150) consumed the entire budget, leaving only ~19 slots for low
priority — causing the observed ~6 month cap.

Medium now selects the 60 oldest-scraped (or never-scraped) dates from
the days 31-180 window; low selects the 30 oldest from days 181-365.
Per-run budget drops from ~365 to ~121 dates, and coverage naturally
cycles through the full year: medium every ~2-3 days, low every ~6-7 days.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/jobs/scrape_booking_rates.py

@@ -1,16 +1,23 @@
 """
 Scheduled Booking.com Rate Scraping Job
 
-Priority-based scheduling for 365-day coverage (all queued daily):
-- High (priority 10): next 30 days
-- Medium (priority 5): days 31-180
-- Low (priority 2): days 181-365
+Tiered scheduling for 365-day coverage:
+- High (priority 10): next 30 days — all 31 dates scraped every day
+- Medium (priority 5): days 31-180 — up to 60 oldest-scraped dates per day
+- Low (priority 2): days 181-365 — up to 30 oldest-scraped dates per day
 
-Queue processes in priority order. If rate-limited/blocked, lower priority
-dates remain queued for the next run.
+Per-run budget: ~121 dates (30 high + 60 medium + 30 low).
+Medium and low tiers select whichever dates in their window have the oldest
+(or missing) scrape data first, naturally distributing coverage across the
+full year without hammering all 365 dates every day.
+
+Approximate refresh cadence:
+- High: daily
+- Medium (150 dates / 60 per day): every ~2-3 days
+- Low (185 dates / 30 per day): every ~6 days
 
 Uses a queue-based approach:
-1. Populate the queue with dates and priorities
+1. Populate the queue with the selected dates and priorities
 2. Process the queue in priority order
 3. Failed dates are retried (up to 3 attempts)
 4. On blocking, the queue pauses and resumes after cooldown
@@ -22,6 +29,7 @@ import logging
 from datetime import date, timedelta
 
 from sqlalchemy import text
+from sqlalchemy.orm import Session
 from database import SyncSessionLocal
 from services.booking_scraper import (
     populate_queue,
@@ -38,32 +46,69 @@ PRIORITY_HIGH = 10    # 0-30 days
 PRIORITY_MEDIUM = 5   # 31-180 days
 PRIORITY_LOW = 2      # 181-365 days
 
+# Per-run date budget for medium and low tiers
+MEDIUM_DATES_PER_RUN = 60   # out of ~150 in window
+LOW_DATES_PER_RUN = 30      # out of ~185 in window
+
 
 def get_high_priority_dates() -> list[date]:
-    """High priority: today + 30 days."""
+    """High priority: all of today through +30 days, scraped every run."""
     today = date.today()
     return [today + timedelta(days=i) for i in range(31)]
 
 
-def get_medium_priority_dates() -> list[date]:
-    """Medium priority: days 31-180."""
-    today = date.today()
-    return [today + timedelta(days=i) for i in range(31, 181)]
+def _get_oldest_scraped_dates(db: Session, start: date, end: date, limit: int) -> list[date]:
+    """
+    From the date window [start, end], return up to `limit` dates ordered by
+    oldest last-scrape first (never-scraped dates come first via NULLS FIRST).
+
+    Joining against booking_com_rates means we naturally pick whichever dates
+    in the window are most stale or have no data yet.
+    """
+    rows = db.execute(
+        text("""
+            SELECT s.rate_date::date AS rate_date,
+                   MAX(r.scraped_at)  AS last_scraped
+            FROM generate_series(:start::date, :end::date, '1 day'::interval) AS s(rate_date)
+            LEFT JOIN booking_com_rates r ON r.rate_date = s.rate_date::date
+            GROUP BY s.rate_date
+            ORDER BY last_scraped ASC NULLS FIRST
+            LIMIT :limit
+        """),
+        {'start': start, 'end': end, 'limit': limit}
+    ).fetchall()
+    return [row.rate_date for row in rows]
 
 
-def get_low_priority_dates() -> list[date]:
-    """Low priority: days 181-365."""
+def get_medium_priority_dates(db: Session, limit: int = MEDIUM_DATES_PER_RUN) -> list[date]:
+    """Medium priority: up to `limit` oldest-scraped dates in the days 31-180 window."""
     today = date.today()
-    return [today + timedelta(days=i) for i in range(181, 366)]
+    return _get_oldest_scraped_dates(
+        db,
+        start=today + timedelta(days=31),
+        end=today + timedelta(days=180),
+        limit=limit,
+    )
+
+
+def get_low_priority_dates(db: Session, limit: int = LOW_DATES_PER_RUN) -> list[date]:
+    """Low priority: up to `limit` oldest-scraped dates in the days 181-365 window."""
+    today = date.today()
+    return _get_oldest_scraped_dates(
+        db,
+        start=today + timedelta(days=181),
+        end=today + timedelta(days=365),
+        limit=limit,
+    )
 
 
 def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
     """
-    For a target date, determine its priority tier and when it will next be scraped.
+    For a target date, determine its priority tier and approximate next scrape date.
 
     Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'.
-    All dates are queued daily, so next scrape is always today (or tomorrow if
-    today's run has passed).
+    High dates are scraped every run; medium/low estimates reflect typical cadence
+    based on the per-run budget vs window size.
     """
     today = date.today()
     offset = (target_date - today).days
@@ -73,13 +118,15 @@ def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
     if offset > 365:
         return ('none', None)
 
-    # All tiers run daily - next scrape is today
     if offset <= 30:
+        # Scraped every day
         return ('high', today)
     elif offset <= 180:
-        return ('medium', today)
+        # ~150 dates, 60/day → ~2-3 day cadence
+        return ('medium', today + timedelta(days=3))
     else:
-        return ('low', today)
+        # ~185 dates, 30/day → ~6-7 day cadence
+        return ('low', today + timedelta(days=7))
 
 
 def run_scheduled_booking_scrape():
@@ -104,10 +151,10 @@ def run_scheduled_booking_scrape():
         cleanup_stale_batches(db, max_age_minutes=120)
         clear_old_queue_items(db, days=3)
 
-        # Gather dates with priorities
+        # Gather dates with priorities — medium/low select oldest-scraped first
         high = get_high_priority_dates()
-        medium = get_medium_priority_dates()
-        low = get_low_priority_dates()
+        medium = get_medium_priority_dates(db)
+        low = get_low_priority_dates(db)
 
         priorities = {}
         for d in high: