rates/backend/services/booking_scraper.py

"""
Booking.com Rate Scraper Service

Main service for scraping competitor rates from booking.com.
Uses pluggable backends (Playwright local, proxy, Apify) via factory pattern.

Features:
- Location-based search (1 query = 40+ hotels)
- Hotel discovery and tier management
- Rate extraction with availability status
- Anti-scrape detection and pause/resume
"""

import logging
import threading
import uuid
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import List, Optional, Dict, Any

from sqlalchemy import text
from sqlalchemy.orm import Session

from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData, RateData, AvailabilityStatus

logger = logging.getLogger(__name__)

# One scrape at a time, process-wide: concurrent Chromium runs starve each
# other of memory (page timeouts → partial results) and double the request
# rate at Booking.com. Guards both manual scrapes and queue processing.
SCRAPE_LOCK = threading.Lock()


def get_scraper_backend(db: Session) -> ScraperBackend:
    """
    Factory to get configured scraper backend.

    Reads backend type from system_config and returns appropriate instance.
    """
    # Get backend configuration
    result = db.execute(
        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_backend'")
    ).fetchone()

    backend_type = result.config_value if result and result.config_value else 'playwright_local'

    if backend_type == 'apify':
        # Future: Apify backend
        raise NotImplementedError("Apify backend not yet implemented")

    if backend_type not in ('playwright_local', 'playwright_proxy'):
        logger.warning(f"Unknown backend type '{backend_type}', falling back to playwright_local")

    # Proxy configuration is managed in system_config (Settings page). When the
    # 'booking_proxy_enabled' key is present the DB is authoritative; otherwise
    # the backend falls back to the BOOKING_PROXY_* environment variables.
    proxy_rows = db.execute(
        text("""
            SELECT config_key, config_value FROM system_config
            WHERE config_key LIKE 'booking_proxy_%'
        """)
    ).fetchall()
    proxy = {row.config_key: row.config_value for row in proxy_rows}

    if 'booking_proxy_enabled' in proxy:
        if proxy.get('booking_proxy_enabled') == 'true':
            return PlaywrightLocalBackend(proxy_config={
                'host': proxy.get('booking_proxy_host', ''),
                'port': proxy.get('booking_proxy_port', '823'),
                'username': proxy.get('booking_proxy_username', ''),
                'password': proxy.get('booking_proxy_password', ''),
                'country': proxy.get('booking_proxy_country', 'gb'),
            })
        # Explicitly disabled in the DB — direct connection, ignore env.
        return PlaywrightLocalBackend(proxy_config={})

    # No DB override — let the backend read BOOKING_PROXY_* env vars.
    return PlaywrightLocalBackend()


def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
    """Get the active scrape location configuration."""
    result = db.execute(
        text("""
            SELECT id, location_name, location_search_url, pages_to_scrape, adults, dest_id
            FROM booking_scrape_config
            WHERE is_active = TRUE
            ORDER BY id
            LIMIT 1
        """)
    ).fetchone()

    if not result:
        return None

    return {
        'id': result.id,
        'location_name': result.location_name,
        'location_search_url': result.location_search_url,
        'pages_to_scrape': result.pages_to_scrape or 2,
        'adults': result.adults or 2,
        'dest_id': result.dest_id,
    }


async def is_scraper_paused(db: Session) -> bool:
    """Check if scraper is currently paused due to blocking."""
    result = db.execute(
        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
    ).fetchone()

    if not result or result.config_value != 'true':
        return False

    # Check if pause period has expired
    pause_until_result = db.execute(
        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'")
    ).fetchone()

    if pause_until_result and pause_until_result.config_value:
        try:
            pause_until = datetime.fromisoformat(pause_until_result.config_value)
            if datetime.now() >= pause_until:
                # Pause expired, reset
                db.execute(
                    text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
                )
                db.commit()
                return False
        except ValueError:
            pass

    return True


async def set_scraper_paused(db: Session, paused: bool, hours: int = 2):
    """Set scraper pause status."""
    db.execute(
        text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_paused'"),
        {'val': 'true' if paused else 'false'}
    )
    if paused:
        pause_until = (datetime.now() + timedelta(hours=hours)).isoformat()
        db.execute(
            text("UPDATE system_config SET config_value = :val WHERE config_key = 'booking_scraper_pause_until'"),
            {'val': pause_until}
        )
    db.commit()


def save_hotel(db: Session, hotel: HotelData) -> int:
    """
    Save or update a hotel in the database.

    Returns the hotel's database ID.
    """
    # Check if hotel exists
    existing = db.execute(
        text("SELECT id FROM booking_com_hotels WHERE booking_com_id = :bid"),
        {'bid': hotel.booking_com_id}
    ).fetchone()

    if existing:
        # Update last_seen_at and any changed fields
        db.execute(
            text("""
                UPDATE booking_com_hotels SET
                    name = COALESCE(:name, name),
                    booking_com_url = COALESCE(:url, booking_com_url),
                    star_rating = COALESCE(:stars, star_rating),
                    review_score = COALESCE(:score, review_score),
                    review_count = COALESCE(:count, review_count),
                    last_seen_at = NOW()
                WHERE booking_com_id = :bid
            """),
            {
                'bid': hotel.booking_com_id,
                'name': hotel.name,
                'url': hotel.booking_com_url,
                'stars': float(hotel.star_rating) if hotel.star_rating else None,
                'score': float(hotel.review_score) if hotel.review_score else None,
                'count': hotel.review_count,
            }
        )
        return existing.id
    else:
        # Insert new hotel (default tier is 'market')
        result = db.execute(
            text("""
                INSERT INTO booking_com_hotels
                (booking_com_id, name, booking_com_url, star_rating, review_score, review_count, tier)
                VALUES (:bid, :name, :url, :stars, :score, :count, 'market')
                RETURNING id
            """),
            {
                'bid': hotel.booking_com_id,
                'name': hotel.name,
                'url': hotel.booking_com_url,
                'stars': float(hotel.star_rating) if hotel.star_rating else None,
                'score': float(hotel.review_score) if hotel.review_score else None,
                'count': hotel.review_count,
            }
        )
        return result.fetchone().id


def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
    """Save a rate to the database."""
    db.execute(
        text("""
            INSERT INTO booking_com_rates
            (hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
             breakfast_included, free_cancellation, no_prepayment, rooms_left, scrape_batch_id)
            VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
                    :breakfast, :cancel, :prepay, :rooms_left, :batch_id)
        """),
        {
            'hotel_id': hotel_id,
            'rate_date': rate.rate_date,
            'status': rate.availability_status.value,
            'rate': float(rate.rate_gross) if rate.rate_gross else None,
            'currency': rate.currency,
            'room_type': rate.room_type,
            'breakfast': rate.breakfast_included,
            'cancel': rate.free_cancellation,
            'prepay': rate.no_prepayment,
            'rooms_left': rate.rooms_left,
            'batch_id': str(batch_id),
        }
    )


def create_scrape_batch(db: Session, scrape_type: str) -> uuid.UUID:
    """Create a new scrape batch log entry."""
    batch_id = uuid.uuid4()
    db.execute(
        text("""
            INSERT INTO booking_scrape_log
            (batch_id, scrape_type, started_at, status)
            VALUES (:batch_id, :scrape_type, NOW(), 'running')
        """),
        {'batch_id': str(batch_id), 'scrape_type': scrape_type}
    )
    db.commit()
    return batch_id


def update_scrape_batch(
    db: Session,
    batch_id: uuid.UUID,
    status: str,
    hotels_found: int = 0,
    rates_scraped: int = 0,
    error_message: str = None,
    blocked: bool = False
):
    """Update scrape batch log with results."""
    db.execute(
        text("""
            UPDATE booking_scrape_log SET
                completed_at = CASE WHEN :status IN ('completed', 'failed', 'blocked') THEN NOW() ELSE NULL END,
                status = :status,
                hotels_found = :hotels,
                rates_scraped = :rates,
                error_message = :error,
                blocked_at = CASE WHEN :blocked THEN NOW() ELSE NULL END,
                resume_after = CASE WHEN :blocked THEN NOW() + INTERVAL '2 hours' ELSE NULL END
            WHERE batch_id = :batch_id
        """),
        {
            'batch_id': str(batch_id),
            'status': status,
            'hotels': hotels_found,
            'rates': rates_scraped,
            'error': error_message,
            'blocked': blocked,
        }
    )
    db.commit()


def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
    """
    Mark any 'running' scrape batches as 'failed' if they've been running
    longer than max_age_minutes. This handles orphaned batches from
    container restarts or crashes.
    """
    result = db.execute(
        text("""
            UPDATE booking_scrape_log SET
                status = 'failed',
                completed_at = NOW(),
                error_message = 'Interrupted (container restart or timeout)'
            WHERE status = 'running'
              AND started_at < NOW() - INTERVAL ':mins minutes'
            RETURNING batch_id
        """.replace(':mins', str(int(max_age_minutes))))
    )
    cleaned = result.fetchall()
    db.commit()
    if cleaned:
        logger.info(f"Cleaned up {len(cleaned)} stale running scrape batch(es)")
    return len(cleaned)


async def scrape_date(
    db: Session,
    rate_date: date,
    backend: ScraperBackend,
    config: Dict[str, Any],
    batch_id: uuid.UUID
) -> Dict[str, Any]:
    """
    Scrape rates for a single date.

    Args:
        db: Database session
        rate_date: Date to scrape rates for
        backend: Scraper backend instance
        config: Scrape configuration
        batch_id: Current batch ID

    Returns:
        Dict with 'success', 'blocked', 'hotels_count', 'rates_count'
    """
    check_in = rate_date
    check_out = rate_date + timedelta(days=1)  # Single night

    result = await backend.scrape_location_search(
        location=config['location_name'],
        check_in=check_in,
        check_out=check_out,
        adults=config['adults'],
        pages=config['pages_to_scrape'],
        dest_id=config.get('dest_id'),
        search_url=config.get('location_search_url'),
    )

    if result.blocked:
        return {
            'success': False,
            'blocked': True,
            'block_reason': result.block_reason,
            'hotels_count': 0,
            'rates_count': 0,
        }

    if not result.success:
        return {
            'success': False,
            'blocked': False,
            'error': result.error_message,
            'hotels_count': 0,
            'rates_count': 0,
        }

    # Save hotels and rates
    hotels_saved = 0
    rates_saved = 0

    for hotel, rate in zip(result.hotels, result.rates):
        if not hotel.booking_com_id:
            continue

        try:
            hotel_id = save_hotel(db, hotel)
            save_rate(db, rate, hotel_id, batch_id)
            hotels_saved += 1
            rates_saved += 1
        except Exception as e:
            logger.warning(f"Error saving hotel/rate: {e}")
            # A failed statement aborts the transaction — roll back so the
            # remaining hotels in this batch can still be saved
            db.rollback()
            continue

    # Flag known hotels absent from this successful scrape as 'not_listed'
    # (sold out or pushed off the search results). Without this their last
    # 'available' rate stays the latest row for the date and reads as a
    # current price, skewing market averages.
    #
    # Absence is only trustworthy when the scrape saw the WHOLE market:
    # a partial scrape (a page timed out / never rendered) or one that saw
    # far fewer hotels than this date normally lists would mark still-listed
    # hotels not_listed and wrongly suppress their last known rates. In
    # those cases keep the rates we did save but skip the flagging, so
    # unseen hotels retain their last known rate and scrape time.
    seen_ids = [h.booking_com_id for h in result.hotels if h.booking_com_id]
    flag_absent = bool(seen_ids)

    if flag_absent and result.pages_ok < result.pages_requested:
        logger.warning(
            f"Partial scrape for {rate_date} ({result.pages_ok}/{result.pages_requested} "
            f"pages ok, {len(seen_ids)} hotels) — skipping not_listed flagging"
        )
        flag_absent = False

    if flag_absent:
        try:
            baseline = db.execute(
                text("""
                    SELECT COUNT(DISTINCT hotel_id)
                    FROM booking_com_rates
                    WHERE rate_date = :rate_date
                      AND availability_status IN ('available', 'sold_out')
                      AND scraped_at > NOW() - INTERVAL '7 days'
                      AND scrape_batch_id != :batch_id
                """),
                {'rate_date': rate_date, 'batch_id': str(batch_id)}
            ).scalar() or 0
            if baseline and len(seen_ids) < baseline * 0.6:
                logger.warning(
                    f"Scrape for {rate_date} saw {len(seen_ids)} hotels vs recent "
                    f"baseline {baseline} — skipping not_listed flagging"
                )
                flag_absent = False
        except Exception as e:
            logger.warning(f"Baseline check failed for {rate_date}: {e}")
            db.rollback()
            flag_absent = False

    if flag_absent:
        try:
            db.execute(
                text("""
                    INSERT INTO booking_com_rates
                        (hotel_id, rate_date, availability_status, rate_gross, scrape_batch_id)
                    SELECT h.id, :rate_date, 'not_listed', NULL, :batch_id
                    FROM booking_com_hotels h
                    WHERE h.is_active = TRUE
                      AND h.booking_com_id != ALL(:seen_ids)
                """),
                {'rate_date': rate_date, 'batch_id': str(batch_id), 'seen_ids': seen_ids}
            )
        except Exception as e:
            logger.warning(f"Error flagging unlisted hotels for {rate_date}: {e}")
            db.rollback()

    db.commit()

    return {
        'success': True,
        'blocked': False,
        'hotels_count': hotels_saved,
        'rates_count': rates_saved,
    }


async def run_manual_scrape(
    db: Session,
    from_date: date,
    to_date: date = None
) -> Dict[str, Any]:
    """
    Run a manual scrape for testing/on-demand use.

    Args:
        db: Database session
        from_date: Start date
        to_date: End date (defaults to from_date for single day)

    Returns:
        Dict with scrape results summary
    """
    if to_date is None:
        to_date = from_date

    if not SCRAPE_LOCK.acquire(blocking=False):
        logger.warning(f"Manual scrape {from_date}..{to_date} refused — another scrape is running")
        return {
            'success': False,
            'already_running': True,
            'error': 'Another scrape is already running. Try again when it finishes.',
        }
    try:
        return await _run_manual_scrape_locked(db, from_date, to_date)
    finally:
        SCRAPE_LOCK.release()


async def _run_manual_scrape_locked(
    db: Session,
    from_date: date,
    to_date: date
) -> Dict[str, Any]:
    # Check if paused
    if await is_scraper_paused(db):
        return {
            'success': False,
            'error': 'Scraper is currently paused due to blocking. Try again later.',
        }

    # Get config
    config = get_scrape_config(db)
    if not config:
        return {
            'success': False,
            'error': 'No scrape location configured. Add a location in settings.',
        }

    # Create batch
    batch_id = create_scrape_batch(db, 'manual')

    # Get backend
    backend = get_scraper_backend(db)

    total_hotels = 0
    total_rates = 0
    dates_completed = 0
    dates_failed = 0

    try:
        current_date = from_date
        while current_date <= to_date:
            logger.info(f"Scraping date: {current_date}")

            result = await scrape_date(db, current_date, backend, config, batch_id)

            if result['blocked']:
                # Blocking detected - pause and exit
                await set_scraper_paused(db, True, hours=2)
                update_scrape_batch(
                    db, batch_id,
                    status='blocked',
                    hotels_found=total_hotels,
                    rates_scraped=total_rates,
                    error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
                    blocked=True
                )
                return {
                    'success': False,
                    'blocked': True,
                    'block_reason': result.get('block_reason'),
                    'dates_completed': dates_completed,
                    'dates_failed': dates_failed,
                    'hotels_found': total_hotels,
                    'rates_scraped': total_rates,
                }

            if result['success']:
                total_hotels += result['hotels_count']
                total_rates += result['rates_count']
                dates_completed += 1
            else:
                dates_failed += 1
                logger.warning(f"Failed to scrape {current_date}: {result.get('error')}")

            current_date += timedelta(days=1)

        # Update batch as completed
        update_scrape_batch(
            db, batch_id,
            status='completed',
            hotels_found=total_hotels,
            rates_scraped=total_rates
        )

        return {
            'success': True,
            'blocked': False,
            'dates_completed': dates_completed,
            'dates_failed': dates_failed,
            'hotels_found': total_hotels,
            'rates_scraped': total_rates,
        }

    except Exception as e:
        logger.error(f"Scrape error: {e}")
        update_scrape_batch(
            db, batch_id,
            status='failed',
            hotels_found=total_hotels,
            rates_scraped=total_rates,
            error_message=str(e)
        )
        return {
            'success': False,
            'error': str(e),
            'dates_completed': dates_completed,
            'dates_failed': dates_failed,
            'hotels_found': total_hotels,
            'rates_scraped': total_rates,
        }
    finally:
        await backend.close()


# ============================================
# QUEUE MANAGEMENT
# ============================================

def populate_queue(db: Session, dates: List[date], priorities: Dict[date, int] = None):
    """
    Add dates to the scrape queue, skipping any already pending/processing.

    Args:
        db: Database session
        dates: Dates to add to the queue
        priorities: Optional priority map (higher = scraped first). Default: 0
    """
    if not dates:
        return 0

    added = 0
    for rate_date in dates:
        priority = (priorities or {}).get(rate_date, 0)
        try:
            db.execute(
                text("""
                    INSERT INTO booking_scrape_queue (rate_date, status, priority)
                    VALUES (:rate_date, 'pending', :priority)
                    ON CONFLICT (rate_date, status) DO UPDATE SET
                        priority = GREATEST(booking_scrape_queue.priority, :priority)
                """),
                {'rate_date': rate_date, 'priority': priority}
            )
            added += 1
        except Exception:
            # Ignore duplicates or constraint issues
            pass

    db.commit()
    logger.info(f"Queue: added/updated {added} dates")
    return added


def get_pending_queue_items(db: Session, limit: int = 50) -> List[Dict[str, Any]]:
    """Get pending queue items ordered by priority (highest first), then date."""
    result = db.execute(
        text("""
            SELECT id, rate_date, priority, attempts, max_attempts
            FROM booking_scrape_queue
            WHERE status = 'pending' AND attempts < max_attempts
            ORDER BY priority DESC, rate_date ASC
            LIMIT :limit
        """),
        {'limit': limit}
    )
    return [dict(row._mapping) for row in result.fetchall()]


def mark_queue_item(db: Session, queue_id: int, status: str, error_message: str = None):
    """Update a queue item's status."""
    if status == 'completed':
        db.execute(
            text("""
                UPDATE booking_scrape_queue SET
                    status = 'completed',
                    completed_at = NOW(),
                    last_attempt_at = NOW(),
                    attempts = attempts + 1
                WHERE id = :id
            """),
            {'id': queue_id}
        )
    elif status == 'failed':
        db.execute(
            text("""
                UPDATE booking_scrape_queue SET
                    status = CASE
                        WHEN attempts + 1 >= max_attempts THEN 'failed'
                        ELSE 'pending'
                    END,
                    last_attempt_at = NOW(),
                    attempts = attempts + 1,
                    error_message = :error
                WHERE id = :id
            """),
            {'id': queue_id, 'error': error_message}
        )
    db.commit()


def clear_old_queue_items(db: Session, days: int = 7):
    """Remove completed/failed queue items older than N days."""
    db.execute(
        text("""
            DELETE FROM booking_scrape_queue
            WHERE status IN ('completed', 'failed')
              AND created_at < NOW() - INTERVAL ':days days'
        """.replace(':days', str(int(days))))
    )
    db.commit()


async def process_queue(db: Session) -> Dict[str, Any]:
    """
    Process pending items from the scrape queue.

    Picks up pending items in priority order, scrapes each date,
    and handles blocking/retries.

    Returns:
        Dict with processing results
    """
    if not SCRAPE_LOCK.acquire(blocking=False):
        logger.warning("Queue processing skipped — another scrape is running")
        return {
            'success': False,
            'already_running': True,
            'error': 'Another scrape is already running.',
        }
    try:
        return await _process_queue_locked(db)
    finally:
        SCRAPE_LOCK.release()


async def _process_queue_locked(db: Session) -> Dict[str, Any]:
    # Check if paused
    if await is_scraper_paused(db):
        return {
            'success': False,
            'error': 'Scraper is currently paused due to blocking.',
        }

    # Get config
    config = get_scrape_config(db)
    if not config:
        return {
            'success': False,
            'error': 'No scrape location configured.',
        }

    # Get pending items
    items = get_pending_queue_items(db, limit=200)
    if not items:
        return {'success': True, 'dates_completed': 0, 'message': 'Queue empty'}

    # Create batch
    batch_id = create_scrape_batch(db, 'scheduled')

    # Update batch with queue count
    db.execute(
        text("UPDATE booking_scrape_log SET dates_queued = :count WHERE batch_id = :bid"),
        {'count': len(items), 'bid': str(batch_id)}
    )
    db.commit()

    # Get backend
    backend = get_scraper_backend(db)

    total_hotels = 0
    total_rates = 0
    dates_completed = 0
    dates_failed = 0

    try:
        for item in items:
            rate_date = item['rate_date']
            queue_id = item['id']

            logger.info(f"Queue processing: {rate_date} (priority={item['priority']}, attempt={item['attempts']+1})")

            result = await scrape_date(db, rate_date, backend, config, batch_id)

            if result['blocked']:
                # Mark this item as failed, pause, and stop
                mark_queue_item(db, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
                await set_scraper_paused(db, True, hours=2)
                update_scrape_batch(
                    db, batch_id,
                    status='blocked',
                    hotels_found=total_hotels,
                    rates_scraped=total_rates,
                    error_message=f"Blocked: {result.get('block_reason', 'unknown')}",
                    blocked=True
                )
                # Update dates counters
                db.execute(
                    text("""
                        UPDATE booking_scrape_log SET
                            dates_completed = :completed,
                            dates_failed = :failed
                        WHERE batch_id = :bid
                    """),
                    {'completed': dates_completed, 'failed': dates_failed + 1, 'bid': str(batch_id)}
                )
                db.commit()
                return {
                    'success': False,
                    'blocked': True,
                    'block_reason': result.get('block_reason'),
                    'dates_completed': dates_completed,
                    'dates_failed': dates_failed + 1,
                    'hotels_found': total_hotels,
                    'rates_scraped': total_rates,
                }

            if result['success']:
                mark_queue_item(db, queue_id, 'completed')
                total_hotels += result['hotels_count']
                total_rates += result['rates_count']
                dates_completed += 1
            else:
                mark_queue_item(db, queue_id, 'failed', result.get('error'))
                dates_failed += 1
                logger.warning(f"Queue: failed to scrape {rate_date}: {result.get('error')}")

        # Update batch as completed
        update_scrape_batch(
            db, batch_id,
            status='completed',
            hotels_found=total_hotels,
            rates_scraped=total_rates
        )
        db.execute(
            text("""
                UPDATE booking_scrape_log SET
                    dates_completed = :completed,
                    dates_failed = :failed
                WHERE batch_id = :bid
            """),
            {'completed': dates_completed, 'failed': dates_failed, 'bid': str(batch_id)}
        )
        db.commit()

        return {
            'success': True,
            'blocked': False,
            'dates_completed': dates_completed,
            'dates_failed': dates_failed,
            'hotels_found': total_hotels,
            'rates_scraped': total_rates,
        }

    except Exception as e:
        logger.error(f"Queue processing error: {e}")
        update_scrape_batch(
            db, batch_id,
            status='failed',
            hotels_found=total_hotels,
            rates_scraped=total_rates,
            error_message=str(e)
        )
        return {
            'success': False,
            'error': str(e),
            'dates_completed': dates_completed,
            'dates_failed': dates_failed,
        }
    finally:
        await backend.close()


def get_competitor_matrix(
    db: Session,
    from_date: date,
    to_date: date,
    include_market: bool = False
) -> List[Dict[str, Any]]:
    """
    Get rate comparison matrix for competitors.

    Args:
        db: Database session
        from_date: Start date
        to_date: End date
        include_market: Include market tier hotels

    Returns:
        List of rate records for matrix display
    """
    tier_filter = "h.tier IN ('own', 'competitor')"
    if include_market:
        tier_filter = "h.tier IN ('own', 'competitor', 'market')"

    result = db.execute(
        text(f"""
            SELECT
                r.rate_date,
                h.id AS hotel_id,
                h.name AS hotel_name,
                h.tier,
                h.display_order,
                h.star_rating,
                h.review_score,
                r.availability_status,
                r.rate_gross,
                r.room_type,
                r.breakfast_included,
                r.free_cancellation,
                r.no_prepayment,
                r.rooms_left,
                r.scraped_at
            FROM booking_latest_rates r
            JOIN booking_com_hotels h ON r.hotel_id = h.id
            WHERE {tier_filter}
              AND h.is_active = TRUE
              AND r.rate_date BETWEEN :from_date AND :to_date
            ORDER BY r.rate_date, h.display_order, h.name
        """),
        {'from_date': from_date, 'to_date': to_date}
    )

    return [dict(row._mapping) for row in result.fetchall()]


def get_hotels_list(db: Session, tier: str = None) -> List[Dict[str, Any]]:
    """
    Get list of discovered hotels.

    Args:
        db: Database session
        tier: Filter by tier ('own', 'competitor', 'market') or None for all

    Returns:
        List of hotel records
    """
    where_clause = "WHERE is_active = TRUE"
    if tier:
        where_clause += f" AND tier = '{tier}'"

    result = db.execute(
        text(f"""
            SELECT
                id, booking_com_id, name, booking_com_url,
                star_rating, review_score, review_count,
                tier, display_order, notes,
                first_seen_at, last_seen_at
            FROM booking_com_hotels
            {where_clause}
            ORDER BY display_order, name
        """)
    )

    return [dict(row._mapping) for row in result.fetchall()]


def update_hotel_tier(db: Session, hotel_id: int, tier: str, display_order: int = None):
    """Update a hotel's tier and display order."""
    if tier not in ('own', 'competitor', 'market'):
        raise ValueError(f"Invalid tier: {tier}")

    params = {'hotel_id': hotel_id, 'tier': tier}
    set_clause = "tier = :tier"

    if display_order is not None:
        set_clause += ", display_order = :order"
        params['order'] = display_order

    db.execute(
        text(f"UPDATE booking_com_hotels SET {set_clause} WHERE id = :hotel_id"),
        params
    )
    db.commit()