hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Add live progress tracking to scrape history

Browse source 
Backend: atomic += increments to hotels_found, rates_scraped,
dates_completed, dates_failed after each date in both worker types
(search-results and hotel-page). Safe for concurrent workers because
PostgreSQL evaluates x = x + delta atomically per statement.
If the container is killed mid-run, the counts already reflect what
was done rather than zeroing out.

Frontend: new Progress column showing X/Y (NN%) of dates completed;
highlighted in gold while the batch is running. scrape-history query
auto-refetches every 5s whenever a running entry is present.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-15 09:09:46 +00:00
parent 1acc817999
commit 312a14b80f
2 changed files with 73 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

55
frontend/src/pages/MarketView.tsx
View file

@ -329,6 +329,8 @@ const SettingsTab: React.FC = () => {
const { data: history } = useQuery<ScrapeHistoryEntry[]>({
queryKey: ['scrape-history'],
queryFn: async () => (await api.get('/competitors/scrape-history?limit=10')).data,
refetchInterval: (query) =>
query.state.data?.some(e => e.status === 'running') ? 5000 : false,
})
const { data: scheduleInfo } = useQuery<ScheduleInfo>({
@ -601,32 +603,45 @@ const SettingsTab: React.FC = () => {
<th style={styles.th}>Type</th>
<th style={styles.th}>Started</th>
<th style={styles.th}>Status</th>
<th style={styles.th}>Progress</th>
<th style={styles.th}>Hotels</th>
<th style={styles.th}>Rates</th>
<th style={styles.th}>Error</th>
</tr>
</thead>
<tbody>
{history.map(entry => (
<tr key={entry.batch_id}>
<td style={styles.td}>{entry.scrape_type}</td>
<td style={styles.td}>{formatDateTime(entry.started_at)}</td>
<td style={styles.td}>
<span style={badgeStyle(
entry.status === 'completed' ? 'success' :
entry.status === 'blocked' ? 'warning' :
entry.status === 'running' ? 'info' : 'error'
)}>
{entry.status}
</span>
</td>
<td style={styles.td}>{entry.hotels_found ?? '-'}</td>
<td style={styles.td}>{entry.rates_scraped ?? '-'}</td>
<td style={mergeStyles(styles.td, { maxWidth: '200px', overflow: 'hidden', textOverflow: 'ellipsis' })}>
{entry.error_message || '-'}
</td>
</tr>
))}
{history.map(entry => {
const queued = entry.dates_queued ?? 0
const done = (entry.dates_completed ?? 0) + (entry.dates_failed ?? 0)
const pct = queued > 0 ? Math.round(done / queued * 100) : null
const progressLabel = queued > 0
? `${done}/${queued}${pct !== null ? ` (${pct}%)` : ''}`
: '-'
const isRunning = entry.status === 'running'
return (
<tr key={entry.batch_id}>
<td style={styles.td}>{entry.scrape_type}</td>
<td style={styles.td}>{formatDateTime(entry.started_at)}</td>
<td style={styles.td}>
<span style={badgeStyle(
entry.status === 'completed' ? 'success' :
entry.status === 'blocked' ? 'warning' :
isRunning ? 'info' : 'error'
)}>
{entry.status}
</span>
</td>
<td style={mergeStyles(styles.td, isRunning ? { color: 'var(--gold)', fontWeight: 600 } : {})}>
{progressLabel}
</td>
<td style={styles.td}>{entry.hotels_found ?? '-'}</td>
<td style={styles.td}>{entry.rates_scraped ?? '-'}</td>
<td style={mergeStyles(styles.td, { maxWidth: '200px', overflow: 'hidden', textOverflow: 'ellipsis' })}>
{entry.error_message || '-'}
</td>
</tr>
)
})}
</tbody>
</table>
</div>