Scraper: force-reset endpoint + watchdog to prevent stuck lock
Root cause: threading.Lock held indefinitely when Playwright browser hangs inside run_in_executor (finally never fires from the async side). Fixes: - _acquire_scrape_lock/_release_scrape_lock track monotonic timestamp - POST /competitors/scrape/reset force-releases the lock and marks any running batch as interrupted (queue rows stay intact for retry) - GET /competitors/status now includes lock_held_seconds - APScheduler watchdog job every 30 min auto-releases if held >3h - Settings → Scraper Proxy tab shows live lock status (green/amber) with a Force Reset button requiring confirmation Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d737940a00
commit
e781b1e8b9
4 changed files with 178 additions and 9 deletions
|
|
@ -68,6 +68,22 @@ async def run_scheduled_parity_check():
|
|||
await loop.run_in_executor(None, run_parity_check)
|
||||
|
||||
|
||||
async def run_scrape_watchdog():
|
||||
"""Auto-release the scrape lock if held for >3 hours (hung Playwright browser)."""
|
||||
from services.booking_scraper import get_lock_status, force_reset_scraper
|
||||
status = get_lock_status()
|
||||
held = status.get("held_seconds")
|
||||
if held and held > 3 * 3600:
|
||||
logger.warning(f"Scrape watchdog: lock held for {held}s — force releasing")
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
force_reset_scraper(db)
|
||||
except Exception as e:
|
||||
logger.error(f"Scrape watchdog reset failed: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def run_scheduled_booking_scrape_async():
|
||||
from jobs.scrape_booking_rates import run_scheduled_booking_scrape
|
||||
import asyncio
|
||||
|
|
@ -131,8 +147,17 @@ def start_scheduler():
|
|||
replace_existing=True,
|
||||
)
|
||||
|
||||
# Scrape lock watchdog — every 30 min; force-releases if held >3 hours
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
scheduler.add_job(
|
||||
run_scrape_watchdog,
|
||||
IntervalTrigger(minutes=30),
|
||||
id='scrape_watchdog',
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
scheduler.start()
|
||||
logger.info(f"Scheduler started: booking scrape at {scrape_hour:02d}:{scrape_minute:02d}, rates fetch at {rates_hour:02d}:{rates_minute:02d}, direct scrape at 06:00, parity check at 06:45")
|
||||
logger.info(f"Scheduler started: booking scrape at {scrape_hour:02d}:{scrape_minute:02d}, rates fetch at {rates_hour:02d}:{rates_minute:02d}, direct scrape at 06:00, parity check at 06:45, watchdog every 30m")
|
||||
|
||||
|
||||
def shutdown_scheduler():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue