Scraper: process-wide lock, one scrape at a time
Concurrent manual scrapes were interleaving (two Chromium sessions on one LXC) causing the page timeouts behind partial results. SCRAPE_LOCK guards run_manual_scrape and process_queue; the trigger endpoint returns 409 when busy, and the frontend keeps the job queued and retries after 30s. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
bf9425ee7e
commit
b712196262
3 changed files with 56 additions and 3 deletions
|
|
@ -12,6 +12,7 @@ Features:
|
|||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
|
|
@ -24,6 +25,11 @@ from .scraper_backends import ScraperBackend, PlaywrightLocalBackend, HotelData,
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# One scrape at a time, process-wide: concurrent Chromium runs starve each
|
||||
# other of memory (page timeouts → partial results) and double the request
|
||||
# rate at Booking.com. Guards both manual scrapes and queue processing.
|
||||
SCRAPE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def get_scraper_backend(db: Session) -> ScraperBackend:
|
||||
"""
|
||||
|
|
@ -445,6 +451,24 @@ async def run_manual_scrape(
|
|||
if to_date is None:
|
||||
to_date = from_date
|
||||
|
||||
if not SCRAPE_LOCK.acquire(blocking=False):
|
||||
logger.warning(f"Manual scrape {from_date}..{to_date} refused — another scrape is running")
|
||||
return {
|
||||
'success': False,
|
||||
'already_running': True,
|
||||
'error': 'Another scrape is already running. Try again when it finishes.',
|
||||
}
|
||||
try:
|
||||
return await _run_manual_scrape_locked(db, from_date, to_date)
|
||||
finally:
|
||||
SCRAPE_LOCK.release()
|
||||
|
||||
|
||||
async def _run_manual_scrape_locked(
|
||||
db: Session,
|
||||
from_date: date,
|
||||
to_date: date
|
||||
) -> Dict[str, Any]:
|
||||
# Check if paused
|
||||
if await is_scraper_paused(db):
|
||||
return {
|
||||
|
|
@ -655,6 +679,20 @@ async def process_queue(db: Session) -> Dict[str, Any]:
|
|||
Returns:
|
||||
Dict with processing results
|
||||
"""
|
||||
if not SCRAPE_LOCK.acquire(blocking=False):
|
||||
logger.warning("Queue processing skipped — another scrape is running")
|
||||
return {
|
||||
'success': False,
|
||||
'already_running': True,
|
||||
'error': 'Another scrape is already running.',
|
||||
}
|
||||
try:
|
||||
return await _process_queue_locked(db)
|
||||
finally:
|
||||
SCRAPE_LOCK.release()
|
||||
|
||||
|
||||
async def _process_queue_locked(db: Session) -> Dict[str, Any]:
|
||||
# Check if paused
|
||||
if await is_scraper_paused(db):
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue