From b47258d47b8485e6700b91c080f8c1de281fffe6 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 20:59:17 +0000 Subject: [PATCH] Fix scrape submission race condition allowing duplicate background tasks Two rapid POST /scrape requests could both pass the lock check before either background task acquired SCRAPE_LOCK, queuing two sequential scrapes for the same date range. The second would hit Cloudflare after the first already succeeded, causing repeated retries. _SCRAPE_PENDING is set on submission and cleared when the task starts, closing the gap between the 409 check and lock acquisition. Co-Authored-By: Claude Sonnet 4.6 --- backend/api/competitors.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/backend/api/competitors.py b/backend/api/competitors.py index 957c215..075c543 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -10,6 +10,8 @@ from sqlalchemy import text from pydantic import BaseModel import logging +import threading + from database import get_db, SyncSessionLocal from auth import get_current_user from services import proxy as proxy_util @@ -17,6 +19,11 @@ from services import proxy as proxy_util router = APIRouter() logger = logging.getLogger(__name__) +# Set when a background task is submitted; cleared when the task starts executing. +# Prevents duplicate scrapes caused by the race between the 409 check and the +# background task acquiring SCRAPE_LOCK. +_SCRAPE_PENDING = threading.Event() + # ============================================ # REQUEST/RESPONSE MODELS @@ -358,6 +365,8 @@ def run_scrape_sync(from_date: date, to_date: date): import asyncio from services.booking_scraper import run_manual_scrape, cleanup_stale_batches + _SCRAPE_PENDING.clear() # Task is now running — allow new submissions to queue + db = SyncSessionLocal() try: # Clean up any stale batches before starting @@ -407,17 +416,13 @@ async def trigger_manual_scrape( if not location_result.fetchone(): raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.") - # Only one scrape at a time — concurrent Chromium runs cause the page - # timeouts that produce partial results - # - # Best-effort early 409: the background task re-acquires the lock and will - # no-op (logging "another scrape is running") if it loses a millisecond- - # window race, so no double-run can slip through here. + # Only one scrape at a time — concurrent Chromium runs cause page timeouts. + # _SCRAPE_PENDING guards the window between submission and lock acquisition. from services.booking_scraper import get_lock_status - if get_lock_status()["locked"]: + if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set(): raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.") - # Start background task + _SCRAPE_PENDING.set() background_tasks.add_task(run_scrape_sync, from_date, to_date) return {