Scraper: process-wide lock, one scrape at a time

Concurrent manual scrapes were interleaving (two Chromium sessions on one
LXC) causing the page timeouts behind partial results. SCRAPE_LOCK guards
run_manual_scrape and process_queue; the trigger endpoint returns 409 when
busy, and the frontend keeps the job queued and retries after 30s.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 16:14:55 +00:00
parent bf9425ee7e
commit b712196262
3 changed files with 56 additions and 3 deletions

View file

@ -12,6 +12,7 @@ Features:
"""
import logging
import threading
import uuid
from datetime import date, datetime, timedelta
from decimal import Decimal
@ -24,6 +25,11 @@ from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData,
logger = logging.getLogger(__name__)
# One scrape at a time, process-wide: concurrent Chromium runs starve each
# other of memory (page timeouts → partial results) and double the request
# rate at Booking.com. Guards both manual scrapes and queue processing.
SCRAPE_LOCK = threading.Lock()
def get_scraper_backend(db: Session) -> ScraperBackend:
"""
@ -445,6 +451,24 @@ async def run_manual_scrape(
if to_date is None:
to_date = from_date
if not SCRAPE_LOCK.acquire(blocking=False):
logger.warning(f"Manual scrape {from_date}..{to_date} refused — another scrape is running")
return {
'success': False,
'already_running': True,
'error': 'Another scrape is already running. Try again when it finishes.',
}
try:
return await _run_manual_scrape_locked(db, from_date, to_date)
finally:
SCRAPE_LOCK.release()
async def _run_manual_scrape_locked(
db: Session,
from_date: date,
to_date: date
) -> Dict[str, Any]:
# Check if paused
if await is_scraper_paused(db):
return {
@ -655,6 +679,20 @@ async def process_queue(db: Session) -> Dict[str, Any]:
Returns:
Dict with processing results
"""
if not SCRAPE_LOCK.acquire(blocking=False):
logger.warning("Queue processing skipped — another scrape is running")
return {
'success': False,
'already_running': True,
'error': 'Another scrape is already running.',
}
try:
return await _process_queue_locked(db)
finally:
SCRAPE_LOCK.release()
async def _process_queue_locked(db: Session) -> Dict[str, Any]:
# Check if paused
if await is_scraper_paused(db):
return {