Flag hotels absent from successful scrapes as 'not_listed' + widen booking_com_id

Location-search results aren't a fixed hotel set — a sold-out hotel drops
out and its last 'available' rate would remain the latest row for that
date, reading as a live price and skewing market averages. On each
successful per-date scrape, insert a NULL-rate 'not_listed' row for every
active hotel missing from the results (skipped if the parse found nothing,
which indicates scraper fault not absence). Failed/blocked scrapes write
nothing, so genuinely-stale data remains distinguishable by scraped_at.

Also: fix the DOW analysis to pick latest-then-filter so a not_listed
latest row drops the date instead of resurfacing an older rate, and widen
booking_com_id to VARCHAR(255) (some Booking slugs exceed 50 chars).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/api/analysis.py

@@ -109,15 +109,17 @@ async def analyse_hotel(
                     ROUND(AVG(rate_gross)::numeric, 2)      AS avg_rate,
                     COUNT(DISTINCT rate_date)               AS date_count
                 FROM (
+                    -- Latest row per date first, THEN filter: a 'not_listed'
+                    -- latest row must drop the date, not resurface an older rate
                     SELECT DISTINCT ON (rate_date)
-                        rate_date, rate_gross
+                        rate_date, rate_gross, availability_status
                     FROM booking_com_rates
                     WHERE hotel_id  = :hid
                       AND rate_date BETWEEN :from_date AND :to_date
-                      AND availability_status = 'available'
-                      AND rate_gross IS NOT NULL
                     ORDER BY rate_date, scraped_at DESC
                 ) latest
+                WHERE availability_status = 'available'
+                  AND rate_gross IS NOT NULL
                 GROUP BY dow, dow_label
                 ORDER BY dow
             """),

backend/schema.sql

@@ -90,7 +90,7 @@ CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending
 
 CREATE TABLE IF NOT EXISTS booking_com_hotels (
     id              SERIAL PRIMARY KEY,
-    booking_com_id  VARCHAR(50) UNIQUE,
+    booking_com_id  VARCHAR(255) UNIQUE,
     name            VARCHAR(255) NOT NULL,
     booking_com_url TEXT,
     star_rating     DECIMAL(2,1),
@@ -124,8 +124,9 @@ CREATE TABLE IF NOT EXISTS booking_com_rates (
     scraped_at          TIMESTAMPTZ DEFAULT NOW()
 );
 
--- Widen room_type on tables created before the VARCHAR(100) → TEXT change
+-- Widen columns on tables created before the type changes
 ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT;
+ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255);
 
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);

backend/services/booking_scraper.py

@@ -354,6 +354,29 @@ async def scrape_date(
             db.rollback()
             continue
 
+    # Flag known hotels absent from this successful scrape as 'not_listed'
+    # (sold out or pushed off the search results). Without this their last
+    # 'available' rate stays the latest row for the date and reads as a
+    # current price, skewing market averages. Skipped when the parse found
+    # nothing at all — that looks like a scraper fault, not real absence.
+    seen_ids = [h.booking_com_id for h in result.hotels if h.booking_com_id]
+    if seen_ids:
+        try:
+            db.execute(
+                text("""
+                    INSERT INTO booking_com_rates
+                        (hotel_id, rate_date, availability_status, rate_gross, scrape_batch_id)
+                    SELECT h.id, :rate_date, 'not_listed', NULL, :batch_id
+                    FROM booking_com_hotels h
+                    WHERE h.is_active = TRUE
+                      AND h.booking_com_id != ALL(:seen_ids)
+                """),
+                {'rate_date': rate_date, 'batch_id': str(batch_id), 'seen_ids': seen_ids}
+            )
+        except Exception as e:
+            logger.warning(f"Error flagging unlisted hotels for {rate_date}: {e}")
+            db.rollback()
+
     db.commit()
 
     return {