From 312a14b80f8de4d1b42ae7b11c190e6b8ff86edb Mon Sep 17 00:00:00 2001
From: jtricerolph <info@tricerolph.com>
Date: Wed, 15 Jul 2026 09:09:46 +0000
Subject: [PATCH] Add live progress tracking to scrape history

Backend: atomic += increments to hotels_found, rates_scraped,
dates_completed, dates_failed after each date in both worker types
(search-results and hotel-page). Safe for concurrent workers because
PostgreSQL evaluates x = x + delta atomically per statement.
If the container is killed mid-run, the counts already reflect what
was done rather than zeroing out.

Frontend: new Progress column showing X/Y (NN%) of dates completed;
highlighted in gold while the batch is running. scrape-history query
auto-refetches every 5s whenever a running entry is present.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 backend/services/booking_scraper.py | 38 ++++++++++++++++++++
 frontend/src/pages/MarketView.tsx   | 55 ++++++++++++++++++-----------
 2 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py
index 533c1ec..3a5f310 100644
--- a/backend/services/booking_scraper.py
+++ b/backend/services/booking_scraper.py
@@ -628,11 +628,16 @@ async def _scrape_hotels_concurrent(
                     if result.get('blocked'):
                         acc['blocked'] += 1
                         acc['failed'] += 1
+                        _increment_batch_progress(wdb, batch_id, delta_failed=1)
                     elif result['success']:
                         acc['rates'] += result['rates_count']
                         acc['completed'] += 1
+                        _increment_batch_progress(wdb, batch_id,
+                                                  delta_rates=result['rates_count'],
+                                                  delta_completed=1)
                     else:
                         acc['failed'] += 1
+                        _increment_batch_progress(wdb, batch_id, delta_failed=1)
         finally:
             try:
                 await backend.close()
@@ -656,6 +661,32 @@ async def _scrape_hotels_concurrent(
     return agg
 
 
+def _increment_batch_progress(
+    db: Session, batch_id: uuid.UUID,
+    delta_hotels: int = 0, delta_rates: int = 0,
+    delta_completed: int = 0, delta_failed: int = 0,
+):
+    """Atomically increment running-batch counters. Safe for concurrent workers
+    because it uses += rather than read-modify-write."""
+    try:
+        db.execute(text("""
+            UPDATE booking_scrape_log
+            SET hotels_found    = COALESCE(hotels_found, 0)    + :dh,
+                rates_scraped   = COALESCE(rates_scraped, 0)   + :dr,
+                dates_completed = COALESCE(dates_completed, 0) + :dc,
+                dates_failed    = COALESCE(dates_failed, 0)    + :df
+            WHERE batch_id = :bid
+        """), {'bid': str(batch_id), 'dh': delta_hotels, 'dr': delta_rates,
+               'dc': delta_completed, 'df': delta_failed})
+        db.commit()
+    except Exception as e:
+        logger.warning(f"Progress update failed for batch {batch_id}: {e}")
+        try:
+            db.rollback()
+        except Exception:
+            pass
+
+
 def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
     """mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
     if queue_id is None:
@@ -701,20 +732,27 @@ async def _scrape_dates_concurrent(
                         pass
                     acc['failed'] += 1
                     _safe_mark_queue(wdb, queue_id, 'failed', str(e))
+                    _increment_batch_progress(wdb, batch_id, delta_failed=1)
                     continue
 
                 if result.get('blocked'):
                     acc['blocked'] += 1
                     acc['failed'] += 1
                     _safe_mark_queue(wdb, queue_id, 'failed', f"Blocked: {result.get('block_reason')}")
+                    _increment_batch_progress(wdb, batch_id, delta_failed=1)
                 elif result['success']:
                     acc['hotels'] += result['hotels_count']
                     acc['rates'] += result['rates_count']
                     acc['completed'] += 1
                     _safe_mark_queue(wdb, queue_id, 'completed')
+                    _increment_batch_progress(wdb, batch_id,
+                                              delta_hotels=result['hotels_count'],
+                                              delta_rates=result['rates_count'],
+                                              delta_completed=1)
                 else:
                     acc['failed'] += 1
                     _safe_mark_queue(wdb, queue_id, 'failed', result.get('error'))
+                    _increment_batch_progress(wdb, batch_id, delta_failed=1)
         finally:
             try:
                 await backend.close()
diff --git a/frontend/src/pages/MarketView.tsx b/frontend/src/pages/MarketView.tsx
index b384e4a..988ee11 100644
--- a/frontend/src/pages/MarketView.tsx
+++ b/frontend/src/pages/MarketView.tsx
@@ -329,6 +329,8 @@ const SettingsTab: React.FC = () => {
   const { data: history } = useQuery<ScrapeHistoryEntry[]>({
     queryKey: ['scrape-history'],
     queryFn: async () => (await api.get('/competitors/scrape-history?limit=10')).data,
+    refetchInterval: (query) =>
+      query.state.data?.some(e => e.status === 'running') ? 5000 : false,
   })
 
   const { data: scheduleInfo } = useQuery<ScheduleInfo>({
@@ -601,32 +603,45 @@ const SettingsTab: React.FC = () => {
                   <th style={styles.th}>Type</th>
                   <th style={styles.th}>Started</th>
                   <th style={styles.th}>Status</th>
+                  <th style={styles.th}>Progress</th>
                   <th style={styles.th}>Hotels</th>
                   <th style={styles.th}>Rates</th>
                   <th style={styles.th}>Error</th>
                 </tr>
               </thead>
               <tbody>
-                {history.map(entry => (
-                  <tr key={entry.batch_id}>
-                    <td style={styles.td}>{entry.scrape_type}</td>
-                    <td style={styles.td}>{formatDateTime(entry.started_at)}</td>
-                    <td style={styles.td}>
-                      <span style={badgeStyle(
-                        entry.status === 'completed' ? 'success' :
-                        entry.status === 'blocked' ? 'warning' :
-                        entry.status === 'running' ? 'info' : 'error'
-                      )}>
-                        {entry.status}
-                      </span>
-                    </td>
-                    <td style={styles.td}>{entry.hotels_found ?? '-'}</td>
-                    <td style={styles.td}>{entry.rates_scraped ?? '-'}</td>
-                    <td style={mergeStyles(styles.td, { maxWidth: '200px', overflow: 'hidden', textOverflow: 'ellipsis' })}>
-                      {entry.error_message || '-'}
-                    </td>
-                  </tr>
-                ))}
+                {history.map(entry => {
+                  const queued = entry.dates_queued ?? 0
+                  const done = (entry.dates_completed ?? 0) + (entry.dates_failed ?? 0)
+                  const pct = queued > 0 ? Math.round(done / queued * 100) : null
+                  const progressLabel = queued > 0
+                    ? `${done}/${queued}${pct !== null ? ` (${pct}%)` : ''}`
+                    : '-'
+                  const isRunning = entry.status === 'running'
+                  return (
+                    <tr key={entry.batch_id}>
+                      <td style={styles.td}>{entry.scrape_type}</td>
+                      <td style={styles.td}>{formatDateTime(entry.started_at)}</td>
+                      <td style={styles.td}>
+                        <span style={badgeStyle(
+                          entry.status === 'completed' ? 'success' :
+                          entry.status === 'blocked' ? 'warning' :
+                          isRunning ? 'info' : 'error'
+                        )}>
+                          {entry.status}
+                        </span>
+                      </td>
+                      <td style={mergeStyles(styles.td, isRunning ? { color: 'var(--gold)', fontWeight: 600 } : {})}>
+                        {progressLabel}
+                      </td>
+                      <td style={styles.td}>{entry.hotels_found ?? '-'}</td>
+                      <td style={styles.td}>{entry.rates_scraped ?? '-'}</td>
+                      <td style={mergeStyles(styles.td, { maxWidth: '200px', overflow: 'hidden', textOverflow: 'ellipsis' })}>
+                        {entry.error_message || '-'}
+                      </td>
+                    </tr>
+                  )
+                })}
               </tbody>
             </table>
           </div>