Scraper: force-reset endpoint + watchdog to prevent stuck lock

Root cause: threading.Lock held indefinitely when Playwright browser
hangs inside run_in_executor (finally never fires from the async side).

Fixes:
- _acquire_scrape_lock/_release_scrape_lock track monotonic timestamp
- POST /competitors/scrape/reset force-releases the lock and marks any
  running batch as interrupted (queue rows stay intact for retry)
- GET /competitors/status now includes lock_held_seconds
- APScheduler watchdog job every 30 min auto-releases if held >3h
- Settings → Scraper Proxy tab shows live lock status (green/amber)
  with a Force Reset button requiring confirmation

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 16:56:35 +00:00
parent d737940a00
commit e781b1e8b9
4 changed files with 178 additions and 9 deletions

View file

@ -68,6 +68,22 @@ async def run_scheduled_parity_check():
await loop.run_in_executor(None, run_parity_check)
async def run_scrape_watchdog():
"""Auto-release the scrape lock if held for >3 hours (hung Playwright browser)."""
from services.booking_scraper import get_lock_status, force_reset_scraper
status = get_lock_status()
held = status.get("held_seconds")
if held and held > 3 * 3600:
logger.warning(f"Scrape watchdog: lock held for {held}s — force releasing")
db = SyncSessionLocal()
try:
force_reset_scraper(db)
except Exception as e:
logger.error(f"Scrape watchdog reset failed: {e}")
finally:
db.close()
async def run_scheduled_booking_scrape_async():
from jobs.scrape_booking_rates import run_scheduled_booking_scrape
import asyncio
@ -131,8 +147,17 @@ def start_scheduler():
replace_existing=True,
)
# Scrape lock watchdog — every 30 min; force-releases if held >3 hours
from apscheduler.triggers.interval import IntervalTrigger
scheduler.add_job(
run_scrape_watchdog,
IntervalTrigger(minutes=30),
id='scrape_watchdog',
replace_existing=True,
)
scheduler.start()
logger.info(f"Scheduler started: booking scrape at {scrape_hour:02d}:{scrape_minute:02d}, rates fetch at {rates_hour:02d}:{rates_minute:02d}, direct scrape at 06:00, parity check at 06:45")
logger.info(f"Scheduler started: booking scrape at {scrape_hour:02d}:{scrape_minute:02d}, rates fetch at {rates_hour:02d}:{rates_minute:02d}, direct scrape at 06:00, parity check at 06:45, watchdog every 30m")
def shutdown_scheduler():