Scraper: force-reset endpoint + watchdog to prevent stuck lock

Root cause: threading.Lock held indefinitely when Playwright browser
hangs inside run_in_executor (finally never fires from the async side).

Fixes:
- _acquire_scrape_lock/_release_scrape_lock track monotonic timestamp
- POST /competitors/scrape/reset force-releases the lock and marks any
  running batch as interrupted (queue rows stay intact for retry)
- GET /competitors/status now includes lock_held_seconds
- APScheduler watchdog job every 30 min auto-releases if held >3h
- Settings → Scraper Proxy tab shows live lock status (green/amber)
  with a Force Reset button requiring confirmation

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/services/booking_scraper.py

@@ -14,6 +14,7 @@ Features:
 import asyncio
 import logging
 import threading
+import time
 import uuid
 from datetime import date, timedelta
 from decimal import Decimal
@@ -32,6 +33,48 @@ logger = logging.getLogger(__name__)
 # other of memory (page timeouts → partial results) and double the request
 # rate at Booking.com. Guards both manual scrapes and queue processing.
 SCRAPE_LOCK = threading.Lock()
+_lock_acquired_at: Optional[float] = None  # monotonic timestamp when lock was acquired
+
+
+def _acquire_scrape_lock() -> bool:
+    global _lock_acquired_at
+    if SCRAPE_LOCK.acquire(blocking=False):
+        _lock_acquired_at = time.monotonic()
+        return True
+    return False
+
+
+def _release_scrape_lock():
+    global _lock_acquired_at
+    _lock_acquired_at = None
+    try:
+        SCRAPE_LOCK.release()
+    except RuntimeError:
+        pass  # already released — race between watchdog and normal release
+
+
+def get_lock_status() -> dict:
+    """Return lock state and how long it has been held (seconds)."""
+    locked = SCRAPE_LOCK.locked()
+    held = int(time.monotonic() - _lock_acquired_at) if (locked and _lock_acquired_at) else None
+    return {"locked": locked, "held_seconds": held}
+
+
+def force_reset_scraper(db: Session) -> dict:
+    """Force-release the scrape lock and tidy up any stuck DB state.
+    Safe to call even when the lock is not held."""
+    was_locked = SCRAPE_LOCK.locked()
+    _release_scrape_lock()
+    # Mark any batch left in 'running' state as interrupted
+    db.execute(text("""
+        UPDATE booking_scrape_log
+        SET status = 'interrupted', completed_at = NOW(),
+            error_message = 'Force-reset by admin'
+        WHERE status = 'running'
+    """))
+    db.commit()
+    logger.warning(f"Scrape lock force-reset (was_locked={was_locked})")
+    return {"was_locked": was_locked, "reset": True}
 
 
 def get_scraper_backend(db: Session) -> ScraperBackend:
@@ -509,7 +552,7 @@ async def run_manual_scrape(
     if to_date is None:
         to_date = from_date
 
-    if not SCRAPE_LOCK.acquire(blocking=False):
+    if not _acquire_scrape_lock():
         logger.warning(f"Manual scrape {from_date}..{to_date} refused — another scrape is running")
         return {
             'success': False,
@@ -519,7 +562,7 @@ async def run_manual_scrape(
     try:
         return await _run_manual_scrape_locked(db, from_date, to_date)
     finally:
-        SCRAPE_LOCK.release()
+        _release_scrape_lock()
 
 
 async def _run_manual_scrape_locked(
@@ -694,7 +737,7 @@ async def process_queue(db: Session) -> Dict[str, Any]:
     Returns:
         Dict with processing results
     """
-    if not SCRAPE_LOCK.acquire(blocking=False):
+    if not _acquire_scrape_lock():
         logger.warning("Queue processing skipped — another scrape is running")
         return {
             'success': False,
@@ -704,7 +747,7 @@ async def process_queue(db: Session) -> Dict[str, Any]:
     try:
         return await _process_queue_locked(db)
     finally:
-        SCRAPE_LOCK.release()
+        _release_scrape_lock()
 
 
 async def _process_queue_locked(db: Session) -> Dict[str, Any]: