From 218b2f45f6cc8de44d5996a6db1be18680fc1650 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Sun, 5 Jul 2026 15:29:06 +0000 Subject: [PATCH] Scraper: don't flag not_listed from partial scrapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A page-load timeout was logged and skipped, so a scrape could 'succeed' with only page 1 of results — and the not_listed flagging then marked every page-2 hotel absent, suppressing their last known rates. - ScraperResult now tracks pages_requested/pages_ok - scrape_date skips not_listed flagging when pages failed or when the scrape saw <60% of the date's 7-day coverage baseline; scraped rates are still saved, unseen hotels keep last known rate + scrape time Co-Authored-By: Claude Fable 5 --- backend/services/booking_scraper.py | 45 +++++++++++++++++-- backend/services/scraper_backends/base.py | 2 + .../scraper_backends/playwright_local.py | 21 +++++++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index e112fcb..7881e4c 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -357,10 +357,49 @@ async def scrape_date( # Flag known hotels absent from this successful scrape as 'not_listed' # (sold out or pushed off the search results). Without this their last # 'available' rate stays the latest row for the date and reads as a - # current price, skewing market averages. Skipped when the parse found - # nothing at all — that looks like a scraper fault, not real absence. + # current price, skewing market averages. + # + # Absence is only trustworthy when the scrape saw the WHOLE market: + # a partial scrape (a page timed out / never rendered) or one that saw + # far fewer hotels than this date normally lists would mark still-listed + # hotels not_listed and wrongly suppress their last known rates. In + # those cases keep the rates we did save but skip the flagging, so + # unseen hotels retain their last known rate and scrape time. seen_ids = [h.booking_com_id for h in result.hotels if h.booking_com_id] - if seen_ids: + flag_absent = bool(seen_ids) + + if flag_absent and result.pages_ok < result.pages_requested: + logger.warning( + f"Partial scrape for {rate_date} ({result.pages_ok}/{result.pages_requested} " + f"pages ok, {len(seen_ids)} hotels) — skipping not_listed flagging" + ) + flag_absent = False + + if flag_absent: + try: + baseline = db.execute( + text(""" + SELECT COUNT(DISTINCT hotel_id) + FROM booking_com_rates + WHERE rate_date = :rate_date + AND availability_status IN ('available', 'sold_out') + AND scraped_at > NOW() - INTERVAL '7 days' + AND scrape_batch_id != :batch_id + """), + {'rate_date': rate_date, 'batch_id': str(batch_id)} + ).scalar() or 0 + if baseline and len(seen_ids) < baseline * 0.6: + logger.warning( + f"Scrape for {rate_date} saw {len(seen_ids)} hotels vs recent " + f"baseline {baseline} — skipping not_listed flagging" + ) + flag_absent = False + except Exception as e: + logger.warning(f"Baseline check failed for {rate_date}: {e}") + db.rollback() + flag_absent = False + + if flag_absent: try: db.execute( text(""" diff --git a/backend/services/scraper_backends/base.py b/backend/services/scraper_backends/base.py index 3d71b0d..271a74c 100644 --- a/backend/services/scraper_backends/base.py +++ b/backend/services/scraper_backends/base.py @@ -59,6 +59,8 @@ class ScraperResult: rates: List[RateData] = field(default_factory=list) error_message: Optional[str] = None page_content_sample: Optional[str] = None # For debugging + pages_requested: int = 0 # Result pages we set out to fetch + pages_ok: int = 0 # Pages that loaded and parsed cleanly class ScraperBackend(ABC): diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index dfe6540..10ff0e1 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -292,6 +292,7 @@ class PlaywrightLocalBackend(ScraperBackend): all_hotels = [] all_rates = [] seen_hotel_ids = set() + pages_ok = 0 context = None page = None @@ -315,10 +316,12 @@ class PlaywrightLocalBackend(ScraperBackend): logger.info(f"Scraping page {page_num + 1}: {url}") + page_loaded = True try: await page.goto(url, wait_until='networkidle', timeout=30000) except Exception as e: logger.warning(f"Page load timeout, continuing: {e}") + page_loaded = False # Check for blocking content = await page.content() @@ -331,7 +334,9 @@ class PlaywrightLocalBackend(ScraperBackend): block_reason=reason, hotels=all_hotels, rates=all_rates, - page_content_sample=content[:1000] + page_content_sample=content[:1000], + pages_requested=pages, + pages_ok=pages_ok, ) # Human-like scrolling @@ -340,6 +345,12 @@ class PlaywrightLocalBackend(ScraperBackend): # Extract data hotels, rates = await self._extract_search_results(page, check_in) + # A page counts as clean if it loaded fully and parsed. + # An empty page 1 means the results never rendered; an empty + # later page can legitimately be the end of the results. + if page_loaded and (hotels or page_num > 0): + pages_ok += 1 + # Deduplicate by booking_com_id for hotel, rate in zip(hotels, rates): if hotel.booking_com_id and hotel.booking_com_id not in seen_hotel_ids: @@ -353,7 +364,9 @@ class PlaywrightLocalBackend(ScraperBackend): success=True, blocked=False, hotels=all_hotels, - rates=all_rates + rates=all_rates, + pages_requested=pages, + pages_ok=pages_ok, ) except Exception as e: @@ -363,7 +376,9 @@ class PlaywrightLocalBackend(ScraperBackend): blocked=False, error_message=str(e), hotels=all_hotels, - rates=all_rates + rates=all_rates, + pages_requested=pages, + pages_ok=pages_ok, ) finally: if page: