Add 'Run Scheduled Scrape Now' button + clarify manual scrape mode

Backend: POST /competitors/scrape/scheduled triggers run_scheduled_booking_scrape
in a background task — repopulates the queue with today's high/medium/low
priority dates and processes it. Returns 400 if scraper is disabled, 409 if
already running.

Frontend: button in the Automatic Schedule card to trigger the scheduled job
manually (useful after an interrupted overnight run). Manual Scrape description
now shows which backend mode is active (hotel-page vs search-results) so it's
clear what the date range scrape actually does.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-15 09:12:14 +00:00
parent 312a14b80f
commit 6503018b99
2 changed files with 79 additions and 1 deletions

View file

@ -361,6 +361,16 @@ async def enable_scraper(
# MANUAL SCRAPE TRIGGER # MANUAL SCRAPE TRIGGER
# ============================================ # ============================================
def _run_scheduled_sync():
"""Run the full scheduled scrape job in a background task."""
_SCRAPE_PENDING.clear() # Task is now running — allow new submissions to queue
try:
from jobs.scrape_booking_rates import run_scheduled_booking_scrape
run_scheduled_booking_scrape()
except Exception as e:
logger.error(f"Triggered scheduled scrape failed: {e}", exc_info=True)
def run_scrape_sync(from_date: date, to_date: date): def run_scrape_sync(from_date: date, to_date: date):
"""Run scrape in sync context for background task.""" """Run scrape in sync context for background task."""
import asyncio import asyncio
@ -434,6 +444,34 @@ async def trigger_manual_scrape(
} }
@router.post("/scrape/scheduled")
async def trigger_scheduled_scrape(
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user),
):
"""Manually trigger the full scheduled scrape job.
Repopulates the queue with today's high/medium/low priority dates and processes it.
Use after an interrupted scheduled run."""
if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
raise HTTPException(status_code=403, detail="manage_scraper capability required")
enabled_row = await db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
)
enabled = (enabled_row.fetchone() or [None])[0]
if enabled != 'true':
raise HTTPException(status_code=400, detail="Scraper is disabled. Enable it first.")
from services.booking_scraper import get_lock_status
if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set():
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
_SCRAPE_PENDING.set()
background_tasks.add_task(_run_scheduled_sync)
return {"status": "started", "message": "Scheduled scrape triggered. Check status for progress."}
@router.post("/scrape/reset") @router.post("/scrape/reset")
async def reset_scraper( async def reset_scraper(
db: AsyncSession = Depends(get_db), db: AsyncSession = Depends(get_db),

View file

@ -386,6 +386,14 @@ const SettingsTab: React.FC = () => {
}, },
}) })
const scheduledScrapeMutation = useMutation({
mutationFn: async () => (await api.post('/competitors/scrape/scheduled')).data,
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
queryClient.invalidateQueries({ queryKey: ['scrape-history'] })
},
})
return ( return (
<div style={styles.settingsGrid}> <div style={styles.settingsGrid}>
{/* Location Configuration */} {/* Location Configuration */}
@ -530,13 +538,45 @@ const SettingsTab: React.FC = () => {
</div> </div>
</div> </div>
)} )}
<div style={{ marginTop: '16px', borderTop: '1px solid var(--border)', paddingTop: '16px' }}>
<p style={styles.cardDescription}>
Re-runs today's full scheduled job repopulates the queue with all high/medium/low priority
dates and processes it. Use after an interrupted scheduled scrape.
</p>
<button
onClick={() => scheduledScrapeMutation.mutate()}
disabled={scheduledScrapeMutation.isPending || !status?.enabled || !status?.location_configured}
style={mergeStyles(
buttonStyle('secondary'),
{ opacity: (!status?.enabled || !status?.location_configured) ? 0.5 : 1 }
)}
>
{scheduledScrapeMutation.isPending ? 'Starting...' : 'Run Scheduled Scrape Now'}
</button>
{scheduledScrapeMutation.isSuccess && (
<p style={{ color: 'var(--success)', fontSize: '13px', marginTop: '8px' }}>
Scheduled scrape triggered. Check status for progress.
</p>
)}
{scheduledScrapeMutation.isError && (
<p style={styles.errorText}>
{(scheduledScrapeMutation.error as any)?.response?.data?.detail || 'Failed to trigger scrape'}
</p>
)}
</div>
</div> </div>
{/* Manual Scrape */} {/* Manual Scrape */}
<div style={styles.card}> <div style={styles.card}>
<h3 style={styles.cardTitle}>Manual Scrape</h3> <h3 style={styles.cardTitle}>Manual Scrape</h3>
<p style={styles.cardDescription}> <p style={styles.cardDescription}>
Trigger a one-off competitor rate scrape for a date range. Runs in background. Scrape a specific date range on demand. Uses the{' '}
<strong>{status?.backend === 'playwright_hotel_page' ? 'hotel-page' : 'search-results'}</strong>{' '}
backend (same as scheduled) {status?.backend === 'playwright_hotel_page'
? 'fetches all rate plans for each tracked hotel.'
: 'searches the configured location and discovers market hotels.'
}
</p> </p>
<div style={styles.formRow}> <div style={styles.formRow}>
<div style={styles.formGroup}> <div style={styles.formGroup}>