From ba6c000903de30ccb6024e66dd2469b03cefff17 Mon Sep 17 00:00:00 2001
From: jtricerolph <info@tricerolph.com>
Date: Sun, 5 Jul 2026 14:44:02 +0000
Subject: [PATCH] Flag hotels absent from successful scrapes as 'not_listed' +
 widen booking_com_id
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Location-search results aren't a fixed hotel set — a sold-out hotel drops
out and its last 'available' rate would remain the latest row for that
date, reading as a live price and skewing market averages. On each
successful per-date scrape, insert a NULL-rate 'not_listed' row for every
active hotel missing from the results (skipped if the parse found nothing,
which indicates scraper fault not absence). Failed/blocked scrapes write
nothing, so genuinely-stale data remains distinguishable by scraped_at.

Also: fix the DOW analysis to pick latest-then-filter so a not_listed
latest row drops the date instead of resurfacing an older rate, and widen
booking_com_id to VARCHAR(255) (some Booking slugs exceed 50 chars).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 backend/api/analysis.py             |  8 +++++---
 backend/schema.sql                  |  5 +++--
 backend/services/booking_scraper.py | 23 +++++++++++++++++++++++
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/backend/api/analysis.py b/backend/api/analysis.py
index 496108d..547b5a1 100644
--- a/backend/api/analysis.py
+++ b/backend/api/analysis.py
@@ -109,15 +109,17 @@ async def analyse_hotel(
                     ROUND(AVG(rate_gross)::numeric, 2)      AS avg_rate,
                     COUNT(DISTINCT rate_date)               AS date_count
                 FROM (
+                    -- Latest row per date first, THEN filter: a 'not_listed'
+                    -- latest row must drop the date, not resurface an older rate
                     SELECT DISTINCT ON (rate_date)
-                        rate_date, rate_gross
+                        rate_date, rate_gross, availability_status
                     FROM booking_com_rates
                     WHERE hotel_id  = :hid
                       AND rate_date BETWEEN :from_date AND :to_date
-                      AND availability_status = 'available'
-                      AND rate_gross IS NOT NULL
                     ORDER BY rate_date, scraped_at DESC
                 ) latest
+                WHERE availability_status = 'available'
+                  AND rate_gross IS NOT NULL
                 GROUP BY dow, dow_label
                 ORDER BY dow
             """),
diff --git a/backend/schema.sql b/backend/schema.sql
index 6515534..85564e6 100644
--- a/backend/schema.sql
+++ b/backend/schema.sql
@@ -90,7 +90,7 @@ CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending
 
 CREATE TABLE IF NOT EXISTS booking_com_hotels (
     id              SERIAL PRIMARY KEY,
-    booking_com_id  VARCHAR(50) UNIQUE,
+    booking_com_id  VARCHAR(255) UNIQUE,
     name            VARCHAR(255) NOT NULL,
     booking_com_url TEXT,
     star_rating     DECIMAL(2,1),
@@ -124,8 +124,9 @@ CREATE TABLE IF NOT EXISTS booking_com_rates (
     scraped_at          TIMESTAMPTZ DEFAULT NOW()
 );
 
--- Widen room_type on tables created before the VARCHAR(100) → TEXT change
+-- Widen columns on tables created before the type changes
 ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT;
+ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255);
 
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py
index b1217b9..e112fcb 100644
--- a/backend/services/booking_scraper.py
+++ b/backend/services/booking_scraper.py
@@ -354,6 +354,29 @@ async def scrape_date(
             db.rollback()
             continue
 
+    # Flag known hotels absent from this successful scrape as 'not_listed'
+    # (sold out or pushed off the search results). Without this their last
+    # 'available' rate stays the latest row for the date and reads as a
+    # current price, skewing market averages. Skipped when the parse found
+    # nothing at all — that looks like a scraper fault, not real absence.
+    seen_ids = [h.booking_com_id for h in result.hotels if h.booking_com_id]
+    if seen_ids:
+        try:
+            db.execute(
+                text("""
+                    INSERT INTO booking_com_rates
+                        (hotel_id, rate_date, availability_status, rate_gross, scrape_batch_id)
+                    SELECT h.id, :rate_date, 'not_listed', NULL, :batch_id
+                    FROM booking_com_hotels h
+                    WHERE h.is_active = TRUE
+                      AND h.booking_com_id != ALL(:seen_ids)
+                """),
+                {'rate_date': rate_date, 'batch_id': str(batch_id), 'seen_ids': seen_ids}
+            )
+        except Exception as e:
+            logger.warning(f"Error flagging unlisted hotels for {rate_date}: {e}")
+            db.rollback()
+
     db.commit()
 
     return {