From 1acc817999b93822e7c42aa780ac180ea68b710a Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Wed, 15 Jul 2026 09:05:20 +0000 Subject: [PATCH] Back-fill hotels/rates counts when marking interrupted scrape batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cleanup_stale_batches and force_reset_scraper were leaving hotels_found and rates_scraped at 0 because update_scrape_batch never ran — the container was killed before it could. Now we subquery booking_com_rates by scrape_batch_id to show what was actually saved before the interruption. Co-Authored-By: Claude Sonnet 4.6 --- backend/services/booking_scraper.py | 42 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 812493a..533c1ec 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -68,11 +68,23 @@ def force_reset_scraper(db: Session) -> dict: Safe to call even when the lock is not held.""" was_locked = SCRAPE_LOCK.locked() _release_scrape_lock() - # Mark any batch left in 'running' state as interrupted + # Mark any batch left in 'running' state as interrupted, back-filling actual counts db.execute(text(""" - UPDATE booking_scrape_log + UPDATE booking_scrape_log bsl SET status = 'interrupted', completed_at = NOW(), - error_message = 'Force-reset by admin' + error_message = 'Force-reset by admin', + hotels_found = ( + SELECT COUNT(DISTINCT hotel_id) + FROM booking_com_rates + WHERE scrape_batch_id = bsl.batch_id::text + AND rate_gross IS NOT NULL + ), + rates_scraped = ( + SELECT COUNT(*) + FROM booking_com_rates + WHERE scrape_batch_id = bsl.batch_id::text + AND rate_gross IS NOT NULL + ) WHERE status = 'running' """)) db.commit() @@ -280,16 +292,30 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60): Mark any 'running' scrape batches as 'failed' if they've been running longer than max_age_minutes. This handles orphaned batches from container restarts or crashes. + Counts are back-filled from booking_com_rates so the history shows what + was actually scraped before the interruption, not zeroes. """ result = db.execute( text(""" - UPDATE booking_scrape_log SET + UPDATE booking_scrape_log bsl SET status = 'failed', completed_at = NOW(), - error_message = 'Interrupted (container restart or timeout)' - WHERE status = 'running' - AND started_at < NOW() - INTERVAL ':mins minutes' - RETURNING batch_id + error_message = 'Interrupted (container restart or timeout)', + hotels_found = ( + SELECT COUNT(DISTINCT hotel_id) + FROM booking_com_rates + WHERE scrape_batch_id = bsl.batch_id::text + AND rate_gross IS NOT NULL + ), + rates_scraped = ( + SELECT COUNT(*) + FROM booking_com_rates + WHERE scrape_batch_id = bsl.batch_id::text + AND rate_gross IS NOT NULL + ) + WHERE bsl.status = 'running' + AND bsl.started_at < NOW() - INTERVAL ':mins minutes' + RETURNING bsl.batch_id """.replace(':mins', str(int(max_age_minutes)))) ) cleaned = result.fetchall()