Add 'Run Scheduled Scrape Now' button + clarify manual scrape mode
Backend: POST /competitors/scrape/scheduled triggers run_scheduled_booking_scrape in a background task — repopulates the queue with today's high/medium/low priority dates and processes it. Returns 400 if scraper is disabled, 409 if already running. Frontend: button in the Automatic Schedule card to trigger the scheduled job manually (useful after an interrupted overnight run). Manual Scrape description now shows which backend mode is active (hotel-page vs search-results) so it's clear what the date range scrape actually does. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
312a14b80f
commit
6503018b99
2 changed files with 79 additions and 1 deletions
|
|
@ -361,6 +361,16 @@ async def enable_scraper(
|
|||
# MANUAL SCRAPE TRIGGER
|
||||
# ============================================
|
||||
|
||||
def _run_scheduled_sync():
|
||||
"""Run the full scheduled scrape job in a background task."""
|
||||
_SCRAPE_PENDING.clear() # Task is now running — allow new submissions to queue
|
||||
try:
|
||||
from jobs.scrape_booking_rates import run_scheduled_booking_scrape
|
||||
run_scheduled_booking_scrape()
|
||||
except Exception as e:
|
||||
logger.error(f"Triggered scheduled scrape failed: {e}", exc_info=True)
|
||||
|
||||
|
||||
def run_scrape_sync(from_date: date, to_date: date):
|
||||
"""Run scrape in sync context for background task."""
|
||||
import asyncio
|
||||
|
|
@ -434,6 +444,34 @@ async def trigger_manual_scrape(
|
|||
}
|
||||
|
||||
|
||||
@router.post("/scrape/scheduled")
|
||||
async def trigger_scheduled_scrape(
|
||||
background_tasks: BackgroundTasks,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Manually trigger the full scheduled scrape job.
|
||||
Repopulates the queue with today's high/medium/low priority dates and processes it.
|
||||
Use after an interrupted scheduled run."""
|
||||
if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
|
||||
raise HTTPException(status_code=403, detail="manage_scraper capability required")
|
||||
|
||||
enabled_row = await db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_enabled'")
|
||||
)
|
||||
enabled = (enabled_row.fetchone() or [None])[0]
|
||||
if enabled != 'true':
|
||||
raise HTTPException(status_code=400, detail="Scraper is disabled. Enable it first.")
|
||||
|
||||
from services.booking_scraper import get_lock_status
|
||||
if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set():
|
||||
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
|
||||
|
||||
_SCRAPE_PENDING.set()
|
||||
background_tasks.add_task(_run_scheduled_sync)
|
||||
return {"status": "started", "message": "Scheduled scrape triggered. Check status for progress."}
|
||||
|
||||
|
||||
@router.post("/scrape/reset")
|
||||
async def reset_scraper(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
|
|
|
|||
|
|
@ -386,6 +386,14 @@ const SettingsTab: React.FC = () => {
|
|||
},
|
||||
})
|
||||
|
||||
const scheduledScrapeMutation = useMutation({
|
||||
mutationFn: async () => (await api.post('/competitors/scrape/scheduled')).data,
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['scrape-history'] })
|
||||
},
|
||||
})
|
||||
|
||||
return (
|
||||
<div style={styles.settingsGrid}>
|
||||
{/* Location Configuration */}
|
||||
|
|
@ -530,13 +538,45 @@ const SettingsTab: React.FC = () => {
|
|||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={{ marginTop: '16px', borderTop: '1px solid var(--border)', paddingTop: '16px' }}>
|
||||
<p style={styles.cardDescription}>
|
||||
Re-runs today's full scheduled job — repopulates the queue with all high/medium/low priority
|
||||
dates and processes it. Use after an interrupted scheduled scrape.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => scheduledScrapeMutation.mutate()}
|
||||
disabled={scheduledScrapeMutation.isPending || !status?.enabled || !status?.location_configured}
|
||||
style={mergeStyles(
|
||||
buttonStyle('secondary'),
|
||||
{ opacity: (!status?.enabled || !status?.location_configured) ? 0.5 : 1 }
|
||||
)}
|
||||
>
|
||||
{scheduledScrapeMutation.isPending ? 'Starting...' : 'Run Scheduled Scrape Now'}
|
||||
</button>
|
||||
{scheduledScrapeMutation.isSuccess && (
|
||||
<p style={{ color: 'var(--success)', fontSize: '13px', marginTop: '8px' }}>
|
||||
Scheduled scrape triggered. Check status for progress.
|
||||
</p>
|
||||
)}
|
||||
{scheduledScrapeMutation.isError && (
|
||||
<p style={styles.errorText}>
|
||||
{(scheduledScrapeMutation.error as any)?.response?.data?.detail || 'Failed to trigger scrape'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Manual Scrape */}
|
||||
<div style={styles.card}>
|
||||
<h3 style={styles.cardTitle}>Manual Scrape</h3>
|
||||
<p style={styles.cardDescription}>
|
||||
Trigger a one-off competitor rate scrape for a date range. Runs in background.
|
||||
Scrape a specific date range on demand. Uses the{' '}
|
||||
<strong>{status?.backend === 'playwright_hotel_page' ? 'hotel-page' : 'search-results'}</strong>{' '}
|
||||
backend (same as scheduled) — {status?.backend === 'playwright_hotel_page'
|
||||
? 'fetches all rate plans for each tracked hotel.'
|
||||
: 'searches the configured location and discovers market hotels.'
|
||||
}
|
||||
</p>
|
||||
<div style={styles.formRow}>
|
||||
<div style={styles.formGroup}>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue