diff --git a/backend/api/analysis.py b/backend/api/analysis.py index 496108d..547b5a1 100644 --- a/backend/api/analysis.py +++ b/backend/api/analysis.py @@ -109,15 +109,17 @@ async def analyse_hotel( ROUND(AVG(rate_gross)::numeric, 2) AS avg_rate, COUNT(DISTINCT rate_date) AS date_count FROM ( + -- Latest row per date first, THEN filter: a 'not_listed' + -- latest row must drop the date, not resurface an older rate SELECT DISTINCT ON (rate_date) - rate_date, rate_gross + rate_date, rate_gross, availability_status FROM booking_com_rates WHERE hotel_id = :hid AND rate_date BETWEEN :from_date AND :to_date - AND availability_status = 'available' - AND rate_gross IS NOT NULL ORDER BY rate_date, scraped_at DESC ) latest + WHERE availability_status = 'available' + AND rate_gross IS NOT NULL GROUP BY dow, dow_label ORDER BY dow """), diff --git a/backend/schema.sql b/backend/schema.sql index 6515534..85564e6 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -90,7 +90,7 @@ CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending CREATE TABLE IF NOT EXISTS booking_com_hotels ( id SERIAL PRIMARY KEY, - booking_com_id VARCHAR(50) UNIQUE, + booking_com_id VARCHAR(255) UNIQUE, name VARCHAR(255) NOT NULL, booking_com_url TEXT, star_rating DECIMAL(2,1), @@ -124,8 +124,9 @@ CREATE TABLE IF NOT EXISTS booking_com_rates ( scraped_at TIMESTAMPTZ DEFAULT NOW() ); --- Widen room_type on tables created before the VARCHAR(100) → TEXT change +-- Widen columns on tables created before the type changes ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT; +ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date); diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index b1217b9..e112fcb 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -354,6 +354,29 @@ async def scrape_date( db.rollback() continue + # Flag known hotels absent from this successful scrape as 'not_listed' + # (sold out or pushed off the search results). Without this their last + # 'available' rate stays the latest row for the date and reads as a + # current price, skewing market averages. Skipped when the parse found + # nothing at all — that looks like a scraper fault, not real absence. + seen_ids = [h.booking_com_id for h in result.hotels if h.booking_com_id] + if seen_ids: + try: + db.execute( + text(""" + INSERT INTO booking_com_rates + (hotel_id, rate_date, availability_status, rate_gross, scrape_batch_id) + SELECT h.id, :rate_date, 'not_listed', NULL, :batch_id + FROM booking_com_hotels h + WHERE h.is_active = TRUE + AND h.booking_com_id != ALL(:seen_ids) + """), + {'rate_date': rate_date, 'batch_id': str(batch_id), 'seen_ids': seen_ids} + ) + except Exception as e: + logger.warning(f"Error flagging unlisted hotels for {rate_date}: {e}") + db.rollback() + db.commit() return {