rates/backend/jobs/scrape_booking_rates.py

"""
Scheduled Booking.com Rate Scraping Job

Priority-based scheduling for 365-day coverage (all queued daily):
- High (priority 10): next 30 days
- Medium (priority 5): days 31-180
- Low (priority 2): days 181-365

Queue processes in priority order. If rate-limited/blocked, lower priority
dates remain queued for the next run.

Uses a queue-based approach:
1. Populate the queue with dates and priorities
2. Process the queue in priority order
3. Failed dates are retried (up to 3 attempts)
4. On blocking, the queue pauses and resumes after cooldown

Schedule: Daily at configurable time (default 05:30)
"""
import asyncio
import logging
from datetime import date, timedelta

from sqlalchemy import text
from database import SyncSessionLocal
from services.booking_scraper import (
    populate_queue,
    process_queue,
    clear_old_queue_items,
    cleanup_stale_batches,
    get_scrape_config,
)

logger = logging.getLogger(__name__)

# Priority levels (higher = processed first)
PRIORITY_HIGH = 10    # 0-30 days
PRIORITY_MEDIUM = 5   # 31-180 days
PRIORITY_LOW = 2      # 181-365 days


def get_high_priority_dates() -> list[date]:
    """High priority: today + 30 days."""
    today = date.today()
    return [today + timedelta(days=i) for i in range(31)]


def get_medium_priority_dates() -> list[date]:
    """Medium priority: days 31-180."""
    today = date.today()
    return [today + timedelta(days=i) for i in range(31, 181)]


def get_low_priority_dates() -> list[date]:
    """Low priority: days 181-365."""
    today = date.today()
    return [today + timedelta(days=i) for i in range(181, 366)]


def compute_next_scrape_for_date(target_date: date) -> tuple[str, date | None]:
    """
    For a target date, determine its priority tier and when it will next be scraped.

    Returns (tier, next_scrape_date) where tier is 'high'/'medium'/'low'/'none'.
    All dates are queued daily, so next scrape is always today (or tomorrow if
    today's run has passed).
    """
    today = date.today()
    offset = (target_date - today).days

    if offset < 0:
        return ('none', None)
    if offset > 365:
        return ('none', None)

    # All tiers run daily - next scrape is today
    if offset <= 30:
        return ('high', today)
    elif offset <= 180:
        return ('medium', today)
    else:
        return ('low', today)


def run_scheduled_booking_scrape():
    """
    Main scheduled job: populate queue with today's dates, then process.
    """
    db = SyncSessionLocal()
    try:
        # Check if scraper is enabled
        result = db.execute(
            text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
        ).fetchone()
        if not result or result.config_value != 'true':
            logger.debug("Scheduled booking scrape skipped (disabled)")
            return

        if not get_scrape_config(db):
            logger.warning("Scheduled booking scrape skipped (no location configured)")
            return

        # Clean up stale running batches and old queue items
        cleanup_stale_batches(db, max_age_minutes=120)
        clear_old_queue_items(db, days=3)

        # Gather dates with priorities
        high = get_high_priority_dates()
        medium = get_medium_priority_dates()
        low = get_low_priority_dates()

        priorities = {}
        for d in high:
            priorities[d] = PRIORITY_HIGH
        for d in medium:
            priorities[d] = max(priorities.get(d, 0), PRIORITY_MEDIUM)
        for d in low:
            priorities[d] = max(priorities.get(d, 0), PRIORITY_LOW)

        all_dates = sorted(priorities.keys())

        if not all_dates:
            logger.info("Scheduled booking scrape: no dates to scrape today")
            return

        logger.info(
            f"Scheduled booking scrape: queuing {len(all_dates)} dates "
            f"(high={len(high)}, medium={len(medium)}, low={len(low)})"
        )

        # Populate queue
        populate_queue(db, all_dates, priorities)

        # Process queue
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        try:
            result = loop.run_until_complete(process_queue(db))
            if result.get('success'):
                logger.info(
                    f"Scheduled booking scrape completed: "
                    f"{result.get('dates_completed', 0)} dates, "
                    f"{result.get('rates_scraped', 0)} rates"
                )
            elif result.get('blocked'):
                logger.warning(
                    f"Scheduled booking scrape blocked: {result.get('block_reason')}. "
                    f"Completed {result.get('dates_completed', 0)} dates. "
                    f"Remaining dates stay queued for retry."
                )
            else:
                logger.error(f"Scheduled booking scrape failed: {result.get('error')}")
        finally:
            loop.close()

    except Exception as e:
        logger.error(f"Scheduled booking scrape error: {e}", exc_info=True)
    finally:
        db.close()


async def run_scheduled_booking_scrape_async():
    """Async wrapper for APScheduler."""
    loop = asyncio.get_event_loop()
    await loop.run_in_executor(None, run_scheduled_booking_scrape)