Scraper: don't flag not_listed from partial scrapes

A page-load timeout was logged and skipped, so a scrape could 'succeed'
with only page 1 of results — and the not_listed flagging then marked
every page-2 hotel absent, suppressing their last known rates.

- ScraperResult now tracks pages_requested/pages_ok
- scrape_date skips not_listed flagging when pages failed or when the
  scrape saw <60% of the date's 7-day coverage baseline; scraped rates
  are still saved, unseen hotels keep last known rate + scrape time

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/services/booking_scraper.py

@@ -357,10 +357,49 @@ async def scrape_date(
     # Flag known hotels absent from this successful scrape as 'not_listed'
     # (sold out or pushed off the search results). Without this their last
     # 'available' rate stays the latest row for the date and reads as a
-    # current price, skewing market averages. Skipped when the parse found
+    # current price, skewing market averages.
-    # nothing at all — that looks like a scraper fault, not real absence.
+    #
+    # Absence is only trustworthy when the scrape saw the WHOLE market:
+    # a partial scrape (a page timed out / never rendered) or one that saw
+    # far fewer hotels than this date normally lists would mark still-listed
+    # hotels not_listed and wrongly suppress their last known rates. In
+    # those cases keep the rates we did save but skip the flagging, so
+    # unseen hotels retain their last known rate and scrape time.
     seen_ids = [h.booking_com_id for h in result.hotels if h.booking_com_id]
-    if seen_ids:
+    flag_absent = bool(seen_ids)
+
+    if flag_absent and result.pages_ok < result.pages_requested:
+        logger.warning(
+            f"Partial scrape for {rate_date} ({result.pages_ok}/{result.pages_requested} "
+            f"pages ok, {len(seen_ids)} hotels) — skipping not_listed flagging"
+        )
+        flag_absent = False
+
+    if flag_absent:
+        try:
+            baseline = db.execute(
+                text("""
+                    SELECT COUNT(DISTINCT hotel_id)
+                    FROM booking_com_rates
+                    WHERE rate_date = :rate_date
+                      AND availability_status IN ('available', 'sold_out')
+                      AND scraped_at > NOW() - INTERVAL '7 days'
+                      AND scrape_batch_id != :batch_id
+                """),
+                {'rate_date': rate_date, 'batch_id': str(batch_id)}
+            ).scalar() or 0
+            if baseline and len(seen_ids) < baseline * 0.6:
+                logger.warning(
+                    f"Scrape for {rate_date} saw {len(seen_ids)} hotels vs recent "
+                    f"baseline {baseline} — skipping not_listed flagging"
+                )
+                flag_absent = False
+        except Exception as e:
+            logger.warning(f"Baseline check failed for {rate_date}: {e}")
+            db.rollback()
+            flag_absent = False
+
+    if flag_absent:
         try:
             db.execute(
                 text("""

backend/services/scraper_backends/base.py

@@ -59,6 +59,8 @@ class ScraperResult:
     rates: List[RateData] = field(default_factory=list)
     error_message: Optional[str] = None
     page_content_sample: Optional[str] = None  # For debugging
+    pages_requested: int = 0             # Result pages we set out to fetch
+    pages_ok: int = 0                    # Pages that loaded and parsed cleanly
 
 
 class ScraperBackend(ABC):

backend/services/scraper_backends/playwright_local.py

@@ -292,6 +292,7 @@ class PlaywrightLocalBackend(ScraperBackend):
         all_hotels = []
         all_rates = []
         seen_hotel_ids = set()
+        pages_ok = 0
 
         context = None
         page = None
@@ -315,10 +316,12 @@ class PlaywrightLocalBackend(ScraperBackend):
 
                 logger.info(f"Scraping page {page_num + 1}: {url}")
 
+                page_loaded = True
                 try:
                     await page.goto(url, wait_until='networkidle', timeout=30000)
                 except Exception as e:
                     logger.warning(f"Page load timeout, continuing: {e}")
+                    page_loaded = False
 
                 # Check for blocking
                 content = await page.content()
@@ -331,7 +334,9 @@ class PlaywrightLocalBackend(ScraperBackend):
                         block_reason=reason,
                         hotels=all_hotels,
                         rates=all_rates,
-                        page_content_sample=content[:1000]
+                        page_content_sample=content[:1000],
+                        pages_requested=pages,
+                        pages_ok=pages_ok,
                     )
 
                 # Human-like scrolling
@@ -340,6 +345,12 @@ class PlaywrightLocalBackend(ScraperBackend):
                 # Extract data
                 hotels, rates = await self._extract_search_results(page, check_in)
 
+                # A page counts as clean if it loaded fully and parsed.
+                # An empty page 1 means the results never rendered; an empty
+                # later page can legitimately be the end of the results.
+                if page_loaded and (hotels or page_num > 0):
+                    pages_ok += 1
+
                 # Deduplicate by booking_com_id
                 for hotel, rate in zip(hotels, rates):
                     if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids:
@@ -353,7 +364,9 @@ class PlaywrightLocalBackend(ScraperBackend):
                 success=True,
                 blocked=False,
                 hotels=all_hotels,
-                rates=all_rates
+                rates=all_rates,
+                pages_requested=pages,
+                pages_ok=pages_ok,
             )
 
         except Exception as e:
@@ -363,7 +376,9 @@ class PlaywrightLocalBackend(ScraperBackend):
                 blocked=False,
                 error_message=str(e),
                 hotels=all_hotels,
-                rates=all_rates
+                rates=all_rates,
+                pages_requested=pages,
+                pages_ok=pages_ok,
             )
         finally:
             if page: