hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add live progress tracking to scrape history

Browse source 
Backend: atomic += increments to hotels_found, rates_scraped,
dates_completed, dates_failed after each date in both worker types
(search-results and hotel-page). Safe for concurrent workers because
PostgreSQL evaluates x = x + delta atomically per statement.
If the container is killed mid-run, the counts already reflect what
was done rather than zeroing out.

Frontend: new Progress column showing X/Y (NN%) of dates completed;
highlighted in gold while the batch is running. scrape-history query
auto-refetches every 5s whenever a running entry is present.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-15 09:09:46 +00:00
parent 1acc817999
commit 312a14b80f
2 changed files with 73 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

38
backend/services/booking_scraper.py
View file

@ -628,11 +628,16 @@ async def _scrape_hotels_concurrent(
if result.get('blocked'):
acc['blocked'] += 1
acc['failed'] += 1
_increment_batch_progress(wdb, batch_id, delta_failed=1)
elif result['success']:
acc['rates'] += result['rates_count']
acc['completed'] += 1
_increment_batch_progress(wdb, batch_id,
delta_rates=result['rates_count'],
delta_completed=1)
else:
acc['failed'] += 1
_increment_batch_progress(wdb, batch_id, delta_failed=1)
finally:
try:
await backend.close()
@ -656,6 +661,32 @@ async def _scrape_hotels_concurrent(
return agg
def _increment_batch_progress(
db: Session, batch_id: uuid.UUID,
delta_hotels: int = 0, delta_rates: int = 0,
delta_completed: int = 0, delta_failed: int = 0,
):
"""Atomically increment running-batch counters. Safe for concurrent workers
because it uses += rather than read-modify-write."""
try:
db.execute(text("""
UPDATE booking_scrape_log
SET hotels_found = COALESCE(hotels_found, 0) + :dh,
rates_scraped = COALESCE(rates_scraped, 0) + :dr,
dates_completed = COALESCE(dates_completed, 0) + :dc,
dates_failed = COALESCE(dates_failed, 0) + :df
WHERE batch_id = :bid
"""), {'bid': str(batch_id), 'dh': delta_hotels, 'dr': delta_rates,
'dc': delta_completed, 'df': delta_failed})
db.commit()
except Exception as e:
logger.warning(f"Progress update failed for batch {batch_id}: {e}")
try:
db.rollback()
except Exception:
pass
def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
"""mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
if queue_id is None:
@ -701,20 +732,27 @@ async def _scrape_dates_concurrent(
pass
acc['failed'] += 1
_safe_mark_queue(wdb, queue_id, 'failed', str(e))
_increment_batch_progress(wdb, batch_id, delta_failed=1)
continue
if result.get('blocked'):
acc['blocked'] += 1
acc['failed'] += 1
_safe_mark_queue(wdb, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
_increment_batch_progress(wdb, batch_id, delta_failed=1)
elif result['success']:
acc['hotels'] += result['hotels_count']
acc['rates'] += result['rates_count']
acc['completed'] += 1
_safe_mark_queue(wdb, queue_id, 'completed')
_increment_batch_progress(wdb, batch_id,
delta_hotels=result['hotels_count'],
delta_rates=result['rates_count'],
delta_completed=1)
else:
acc['failed'] += 1
_safe_mark_queue(wdb, queue_id, 'failed', result.get('error'))
_increment_batch_progress(wdb, batch_id, delta_failed=1)
finally:
try:
await backend.close()