Scraper: force-reset endpoint + watchdog to prevent stuck lock

Root cause: threading.Lock held indefinitely when Playwright browser
hangs inside run_in_executor (finally never fires from the async side).

Fixes:
- _acquire_scrape_lock/_release_scrape_lock track monotonic timestamp
- POST /competitors/scrape/reset force-releases the lock and marks any
  running batch as interrupted (queue rows stay intact for retry)
- GET /competitors/status now includes lock_held_seconds
- APScheduler watchdog job every 30 min auto-releases if held >3h
- Settings → Scraper Proxy tab shows live lock status (green/amber)
  with a Force Reset button requiring confirmation

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 16:56:35 +00:00
parent d737940a00
commit e781b1e8b9
4 changed files with 178 additions and 9 deletions

View file

@ -1,7 +1,7 @@
import { useState, useEffect } from 'react'
import { useParams, useNavigate } from 'react-router-dom'
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
import { Save, RefreshCw, Database, Clock, BedDouble, ChevronUp, ChevronDown, Network, ShieldCheck } from 'lucide-react'
import { Save, RefreshCw, Database, Clock, BedDouble, ChevronUp, ChevronDown, Network, ShieldCheck, AlertTriangle } from 'lucide-react'
import api from '../api'
const TABS = [
@ -488,6 +488,83 @@ function ProxyTab() {
</div>
</div>
</div>
<ScraperLockCard />
</div>
)
}
// ─── Scraper Lock Status Card ─────────────────────────────────────────────────
function ScraperLockCard() {
const qc = useQueryClient()
const { data: status, isLoading } = useQuery<{ lock_held_seconds: number | null }>({
queryKey: ['scraper-status-lock'],
queryFn: () => api.get('/competitors/status').then(r => r.data),
refetchInterval: 15000,
})
const resetM = useMutation({
mutationFn: () => api.post('/competitors/scrape/reset').then(r => r.data),
onSuccess: () => qc.invalidateQueries({ queryKey: ['scraper-status-lock'] }),
})
const held = status?.lock_held_seconds ?? null
const isStuck = held !== null && held > 0
const heldStr = held
? held >= 3600
? `${Math.floor(held / 3600)}h ${Math.floor((held % 3600) / 60)}m`
: held >= 60
? `${Math.floor(held / 60)}m ${held % 60}s`
: `${held}s`
: null
return (
<div className="card" style={{ marginTop: 20 }}>
<div className="card-header">Scraper Lock</div>
<div style={{ padding: '16px 20px' }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 12, marginBottom: 12 }}>
<span style={{
display: 'inline-flex', alignItems: 'center', gap: 6,
padding: '4px 10px', borderRadius: 20, fontSize: 13, fontWeight: 600,
background: isStuck ? '#fef3c7' : '#dcfce7',
color: isStuck ? '#d97706' : '#16a34a',
}}>
{isStuck
? <><AlertTriangle size={13} strokeWidth={1.75} /> Locked ({heldStr})</>
: <><ShieldCheck size={13} strokeWidth={1.75} /> Idle</>}
</span>
{isStuck && (
<span style={{ fontSize: 12, color: 'var(--text-mid)' }}>
A scrape may be stuck. The watchdog will auto-release after 3 hours.
</span>
)}
</div>
<p style={{ fontSize: 13, color: 'var(--text-mid)', marginBottom: 12 }}>
Force-release the scrape lock if it is stuck (e.g. a hung Playwright browser).
Any in-progress scrape batch will be marked interrupted; the queue remains intact
and will retry on the next scheduled run or manual trigger.
</p>
<button
className="btn btn-sm"
style={{ background: '#fee2e2', color: '#dc2626', border: '1px solid #fca5a5' }}
disabled={resetM.isPending || isLoading}
onClick={() => {
if (window.confirm('Force-reset the scraper lock? Any running scrape will be interrupted.')) {
resetM.mutate()
}
}}
>
<RefreshCw size={13} strokeWidth={1.75} style={resetM.isPending ? { animation: 'spin 1.5s linear infinite' } : undefined} />
{resetM.isPending ? 'Resetting…' : 'Force Reset Lock'}
</button>
{resetM.isSuccess && (
<span style={{ marginLeft: 12, fontSize: 12, color: 'var(--success)' }}>
Reset was {(resetM.data as any)?.was_locked ? 'locked' : 'already idle'}
</span>
)}
</div>
</div>
)
}