hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Back-fill hotels/rates counts when marking interrupted scrape batches

Browse source 
cleanup_stale_batches and force_reset_scraper were leaving hotels_found
and rates_scraped at 0 because update_scrape_batch never ran — the
container was killed before it could. Now we subquery booking_com_rates
by scrape_batch_id to show what was actually saved before the interruption.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-15 09:05:20 +00:00
parent 9fa4c15081
commit 1acc817999
1 changed files with 34 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

42
backend/services/booking_scraper.py
View file

@ -68,11 +68,23 @@ def force_reset_scraper(db: Session) -> dict:
Safe to call even when the lock is not held."""
was_locked = SCRAPE_LOCK.locked()
_release_scrape_lock()
# Mark any batch left in 'running' state as interrupted
# Mark any batch left in 'running' state as interrupted, back-filling actual counts
db.execute(text("""
UPDATE booking_scrape_log
UPDATE booking_scrape_log bsl
SET status = 'interrupted', completed_at = NOW(),
error_message = 'Force-reset by admin'
error_message = 'Force-reset by admin',
hotels_found = (
SELECT COUNT(DISTINCT hotel_id)
FROM booking_com_rates
WHERE scrape_batch_id = bsl.batch_id::text
AND rate_gross IS NOT NULL
),
rates_scraped = (
SELECT COUNT(*)
FROM booking_com_rates
WHERE scrape_batch_id = bsl.batch_id::text
AND rate_gross IS NOT NULL
)
WHERE status = 'running'
"""))
db.commit()
@ -280,16 +292,30 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
Mark any 'running' scrape batches as 'failed' if they've been running
longer than max_age_minutes. This handles orphaned batches from
container restarts or crashes.
Counts are back-filled from booking_com_rates so the history shows what
was actually scraped before the interruption, not zeroes.
"""
result = db.execute(
text("""
UPDATE booking_scrape_log SET
UPDATE booking_scrape_log bsl SET
status = 'failed',
completed_at = NOW(),
error_message = 'Interrupted (container restart or timeout)'
WHERE status = 'running'
AND started_at < NOW() - INTERVAL ':mins minutes'
RETURNING batch_id
error_message = 'Interrupted (container restart or timeout)',
hotels_found = (
SELECT COUNT(DISTINCT hotel_id)
FROM booking_com_rates
WHERE scrape_batch_id = bsl.batch_id::text
AND rate_gross IS NOT NULL
),
rates_scraped = (
SELECT COUNT(*)
FROM booking_com_rates
WHERE scrape_batch_id = bsl.batch_id::text
AND rate_gross IS NOT NULL
)
WHERE bsl.status = 'running'
AND bsl.started_at < NOW() - INTERVAL ':mins minutes'
RETURNING bsl.batch_id
""".replace(':mins', str(int(max_age_minutes))))
)
cleaned = result.fetchall()