Scraper: force-reset endpoint + watchdog to prevent stuck lock

Root cause: threading.Lock held indefinitely when Playwright browser
hangs inside run_in_executor (finally never fires from the async side).

Fixes:
- _acquire_scrape_lock/_release_scrape_lock track monotonic timestamp
- POST /competitors/scrape/reset force-releases the lock and marks any
  running batch as interrupted (queue rows stay intact for retry)
- GET /competitors/status now includes lock_held_seconds
- APScheduler watchdog job every 30 min auto-releases if held >3h
- Settings → Scraper Proxy tab shows live lock status (green/amber)
  with a Force Reset button requiring confirmation

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 16:56:35 +00:00
parent d737940a00
commit e781b1e8b9
4 changed files with 178 additions and 9 deletions

View file

@ -79,6 +79,7 @@ class ScraperStatusResponse(BaseModel):
location_configured: bool
location_name: Optional[str]
last_scrape: Optional[dict]
lock_held_seconds: Optional[int] = None
# ============================================
@ -133,12 +134,16 @@ async def get_scraper_status(
'error_message': last_scrape_row.error_message,
}
from services.booking_scraper import get_lock_status
lock = get_lock_status()
return ScraperStatusResponse(
enabled=config.get('booking_scraper_enabled', 'false') == 'true',
backend=config.get('booking_scraper_backend', 'playwright_local'),
location_configured=location_row is not None,
location_name=location_row.location_name if location_row else None,
last_scrape=last_scrape
last_scrape=last_scrape,
lock_held_seconds=lock["held_seconds"],
)
@ -408,8 +413,8 @@ async def trigger_manual_scrape(
# Best-effort early 409: the background task re-acquires the lock and will
# no-op (logging "another scrape is running") if it loses a millisecond-
# window race, so no double-run can slip through here.
from services.booking_scraper import SCRAPE_LOCK
if SCRAPE_LOCK.locked():
from services.booking_scraper import get_lock_status
if get_lock_status()["locked"]:
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
# Start background task
@ -423,6 +428,25 @@ async def trigger_manual_scrape(
}
@router.post("/scrape/reset")
async def reset_scraper(
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(get_current_user),
):
"""Force-release the scrape lock and mark any stuck running batches as interrupted.
Use when the scraper is stuck and the 409 won't clear on its own."""
if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
raise HTTPException(status_code=403, detail="manage_scraper capability required")
from services.booking_scraper import force_reset_scraper
sync_db = SyncSessionLocal()
try:
result = force_reset_scraper(sync_db)
finally:
sync_db.close()
return result
# ============================================
# HOTELS MANAGEMENT
# ============================================