From 6503018b997e9344804435e499fbbfdabed8f000 Mon Sep 17 00:00:00 2001
From: jtricerolph <info@tricerolph.com>
Date: Wed, 15 Jul 2026 09:12:14 +0000
Subject: [PATCH] Add 'Run Scheduled Scrape Now' button + clarify manual scrape
 mode
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Backend: POST /competitors/scrape/scheduled triggers run_scheduled_booking_scrape
in a background task — repopulates the queue with today's high/medium/low
priority dates and processes it. Returns 400 if scraper is disabled, 409 if
already running.

Frontend: button in the Automatic Schedule card to trigger the scheduled job
manually (useful after an interrupted overnight run). Manual Scrape description
now shows which backend mode is active (hotel-page vs search-results) so it's
clear what the date range scrape actually does.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 backend/api/competitors.py        | 38 ++++++++++++++++++++++++++++
 frontend/src/pages/MarketView.tsx | 42 ++++++++++++++++++++++++++++++-
 2 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/backend/api/competitors.py b/backend/api/competitors.py
index 3379f33..7439ca5 100644
--- a/backend/api/competitors.py
+++ b/backend/api/competitors.py
@@ -361,6 +361,16 @@ async def enable_scraper(
 # MANUAL SCRAPE TRIGGER
 # ============================================
 
+def _run_scheduled_sync():
+    """Run the full scheduled scrape job in a background task."""
+    _SCRAPE_PENDING.clear()  # Task is now running — allow new submissions to queue
+    try:
+        from jobs.scrape_booking_rates import run_scheduled_booking_scrape
+        run_scheduled_booking_scrape()
+    except Exception as e:
+        logger.error(f"Triggered scheduled scrape failed: {e}", exc_info=True)
+
+
 def run_scrape_sync(from_date: date, to_date: date):
     """Run scrape in sync context for background task."""
     import asyncio
@@ -434,6 +444,34 @@ async def trigger_manual_scrape(
     }
 
 
+@router.post("/scrape/scheduled")
+async def trigger_scheduled_scrape(
+    background_tasks: BackgroundTasks,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user),
+):
+    """Manually trigger the full scheduled scrape job.
+    Repopulates the queue with today's high/medium/low priority dates and processes it.
+    Use after an interrupted scheduled run."""
+    if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
+        raise HTTPException(status_code=403, detail="manage_scraper capability required")
+
+    enabled_row = await db.execute(
+        text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
+    )
+    enabled = (enabled_row.fetchone() or [None])[0]
+    if enabled != 'true':
+        raise HTTPException(status_code=400, detail="Scraper is disabled. Enable it first.")
+
+    from services.booking_scraper import get_lock_status
+    if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set():
+        raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
+
+    _SCRAPE_PENDING.set()
+    background_tasks.add_task(_run_scheduled_sync)
+    return {"status": "started", "message": "Scheduled scrape triggered. Check status for progress."}
+
+
 @router.post("/scrape/reset")
 async def reset_scraper(
     db: AsyncSession = Depends(get_db),
diff --git a/frontend/src/pages/MarketView.tsx b/frontend/src/pages/MarketView.tsx
index 988ee11..306f91b 100644
--- a/frontend/src/pages/MarketView.tsx
+++ b/frontend/src/pages/MarketView.tsx
@@ -386,6 +386,14 @@ const SettingsTab: React.FC = () => {
     },
   })
 
+  const scheduledScrapeMutation = useMutation({
+    mutationFn: async () => (await api.post('/competitors/scrape/scheduled')).data,
+    onSuccess: () => {
+      queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
+      queryClient.invalidateQueries({ queryKey: ['scrape-history'] })
+    },
+  })
+
   return (
     <div style={styles.settingsGrid}>
       {/* Location Configuration */}
@@ -530,13 +538,45 @@ const SettingsTab: React.FC = () => {
             </div>
           </div>
         )}
+
+        <div style={{ marginTop: '16px', borderTop: '1px solid var(--border)', paddingTop: '16px' }}>
+          <p style={styles.cardDescription}>
+            Re-runs today's full scheduled job — repopulates the queue with all high/medium/low priority
+            dates and processes it. Use after an interrupted scheduled scrape.
+          </p>
+          <button
+            onClick={() => scheduledScrapeMutation.mutate()}
+            disabled={scheduledScrapeMutation.isPending || !status?.enabled || !status?.location_configured}
+            style={mergeStyles(
+              buttonStyle('secondary'),
+              { opacity: (!status?.enabled || !status?.location_configured) ? 0.5 : 1 }
+            )}
+          >
+            {scheduledScrapeMutation.isPending ? 'Starting...' : 'Run Scheduled Scrape Now'}
+          </button>
+          {scheduledScrapeMutation.isSuccess && (
+            <p style={{ color: 'var(--success)', fontSize: '13px', marginTop: '8px' }}>
+              Scheduled scrape triggered. Check status for progress.
+            </p>
+          )}
+          {scheduledScrapeMutation.isError && (
+            <p style={styles.errorText}>
+              {(scheduledScrapeMutation.error as any)?.response?.data?.detail || 'Failed to trigger scrape'}
+            </p>
+          )}
+        </div>
       </div>
 
       {/* Manual Scrape */}
       <div style={styles.card}>
         <h3 style={styles.cardTitle}>Manual Scrape</h3>
         <p style={styles.cardDescription}>
-          Trigger a one-off competitor rate scrape for a date range. Runs in background.
+          Scrape a specific date range on demand. Uses the{' '}
+          <strong>{status?.backend === 'playwright_hotel_page' ? 'hotel-page' : 'search-results'}</strong>{' '}
+          backend (same as scheduled) — {status?.backend === 'playwright_hotel_page'
+            ? 'fetches all rate plans for each tracked hotel.'
+            : 'searches the configured location and discovers market hotels.'
+          }
         </p>
         <div style={styles.formRow}>
           <div style={styles.formGroup}>