hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add live progress tracking to scrape history

Browse source 
Backend: atomic += increments to hotels_found, rates_scraped,
dates_completed, dates_failed after each date in both worker types
(search-results and hotel-page). Safe for concurrent workers because
PostgreSQL evaluates x = x + delta atomically per statement.
If the container is killed mid-run, the counts already reflect what
was done rather than zeroing out.

Frontend: new Progress column showing X/Y (NN%) of dates completed;
highlighted in gold while the batch is running. scrape-history query
auto-refetches every 5s whenever a running entry is present.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-15 09:09:46 +00:00
parent 1acc817999
commit 312a14b80f
2 changed files with 73 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

38
backend/services/booking_scraper.py
View file

@ -628,11 +628,16 @@ async def _scrape_hotels_concurrent(
if result.get('blocked'):
acc['blocked'] += 1
acc['failed'] += 1
_increment_batch_progress(wdb, batch_id, delta_failed=1)
elif result['success']:
acc['rates'] += result['rates_count']
acc['completed'] += 1
_increment_batch_progress(wdb, batch_id,
delta_rates=result['rates_count'],
delta_completed=1)
else:
acc['failed'] += 1
_increment_batch_progress(wdb, batch_id, delta_failed=1)
finally:
try:
await backend.close()
@ -656,6 +661,32 @@ async def _scrape_hotels_concurrent(
return agg
def _increment_batch_progress(
db: Session, batch_id: uuid.UUID,
delta_hotels: int = 0, delta_rates: int = 0,
delta_completed: int = 0, delta_failed: int = 0,
):
"""Atomically increment running-batch counters. Safe for concurrent workers
because it uses += rather than read-modify-write."""
try:
db.execute(text("""
UPDATE booking_scrape_log
SET hotels_found = COALESCE(hotels_found, 0) + :dh,
rates_scraped = COALESCE(rates_scraped, 0) + :dr,
dates_completed = COALESCE(dates_completed, 0) + :dc,
dates_failed = COALESCE(dates_failed, 0) + :df
WHERE batch_id = :bid
"""), {'bid': str(batch_id), 'dh': delta_hotels, 'dr': delta_rates,
'dc': delta_completed, 'df': delta_failed})
db.commit()
except Exception as e:
logger.warning(f"Progress update failed for batch {batch_id}: {e}")
try:
db.rollback()
except Exception:
pass
def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
"""mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
if queue_id is None:
@ -701,20 +732,27 @@ async def _scrape_dates_concurrent(
pass
acc['failed'] += 1
_safe_mark_queue(wdb, queue_id, 'failed', str(e))
_increment_batch_progress(wdb, batch_id, delta_failed=1)
continue
if result.get('blocked'):
acc['blocked'] += 1
acc['failed'] += 1
_safe_mark_queue(wdb, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
_increment_batch_progress(wdb, batch_id, delta_failed=1)
elif result['success']:
acc['hotels'] += result['hotels_count']
acc['rates'] += result['rates_count']
acc['completed'] += 1
_safe_mark_queue(wdb, queue_id, 'completed')
_increment_batch_progress(wdb, batch_id,
delta_hotels=result['hotels_count'],
delta_rates=result['rates_count'],
delta_completed=1)
else:
acc['failed'] += 1
_safe_mark_queue(wdb, queue_id, 'failed', result.get('error'))
_increment_batch_progress(wdb, batch_id, delta_failed=1)
finally:
try:
await backend.close()

55
frontend/src/pages/MarketView.tsx
View file

@ -329,6 +329,8 @@ const SettingsTab: React.FC = () => {
const { data: history } = useQuery<ScrapeHistoryEntry[]>({
queryKey: ['scrape-history'],
queryFn: async () => (await api.get('/competitors/scrape-history?limit=10')).data,
refetchInterval: (query) =>
query.state.data?.some(e => e.status === 'running') ? 5000 : false,
})
const { data: scheduleInfo } = useQuery<ScheduleInfo>({
@ -601,32 +603,45 @@ const SettingsTab: React.FC = () => {
<th style={styles.th}>Type</th>
<th style={styles.th}>Started</th>
<th style={styles.th}>Status</th>
<th style={styles.th}>Progress</th>
<th style={styles.th}>Hotels</th>
<th style={styles.th}>Rates</th>
<th style={styles.th}>Error</th>
</tr>
</thead>
<tbody>
{history.map(entry => (
<tr key={entry.batch_id}>
<td style={styles.td}>{entry.scrape_type}</td>
<td style={styles.td}>{formatDateTime(entry.started_at)}</td>
<td style={styles.td}>
<span style={badgeStyle(
entry.status === 'completed' ? 'success' :
entry.status === 'blocked' ? 'warning' :
entry.status === 'running' ? 'info' : 'error'
)}>
{entry.status}
</span>
</td>
<td style={styles.td}>{entry.hotels_found ?? '-'}</td>
<td style={styles.td}>{entry.rates_scraped ?? '-'}</td>
<td style={mergeStyles(styles.td, { maxWidth: '200px', overflow: 'hidden', textOverflow: 'ellipsis' })}>
{entry.error_message || '-'}
</td>
</tr>
))}
{history.map(entry => {
const queued = entry.dates_queued ?? 0
const done = (entry.dates_completed ?? 0) + (entry.dates_failed ?? 0)
const pct = queued > 0 ? Math.round(done / queued * 100) : null
const progressLabel = queued > 0
? `${done}/${queued}${pct !== null ? ` (${pct}%)` : ''}`
: '-'
const isRunning = entry.status === 'running'
return (
<tr key={entry.batch_id}>
<td style={styles.td}>{entry.scrape_type}</td>
<td style={styles.td}>{formatDateTime(entry.started_at)}</td>
<td style={styles.td}>
<span style={badgeStyle(
entry.status === 'completed' ? 'success' :
entry.status === 'blocked' ? 'warning' :
isRunning ? 'info' : 'error'
)}>
{entry.status}
</span>
</td>
<td style={mergeStyles(styles.td, isRunning ? { color: 'var(--gold)', fontWeight: 600 } : {})}>
{progressLabel}
</td>
<td style={styles.td}>{entry.hotels_found ?? '-'}</td>
<td style={styles.td}>{entry.rates_scraped ?? '-'}</td>
<td style={mergeStyles(styles.td, { maxWidth: '200px', overflow: 'hidden', textOverflow: 'ellipsis' })}>
{entry.error_message || '-'}
</td>
</tr>
)
})}
</tbody>
</table>
</div>