hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Scraper: process-wide lock, one scrape at a time

Browse source 
Concurrent manual scrapes were interleaving (two Chromium sessions on one
LXC) causing the page timeouts behind partial results. SCRAPE_LOCK guards
run_manual_scrape and process_queue; the trigger endpoint returns 409 when
busy, and the frontend keeps the job queued and retries after 30s.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 16:14:55 +00:00
parent bf9425ee7e
commit b712196262
3 changed files with 56 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

15
frontend/src/pages/MarketView.tsx
View file

@ -949,6 +949,8 @@ const RateMatrixTab: React.FC = () => {
// batch finishes, refetch the matrix, then start the next job.
const [scrapeQueue, setScrapeQueue] = useState<ScrapeJob[]>([])
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number, timeoutMs: number } | null>(null)
const [dispatchHoldUntil, setDispatchHoldUntil] = useState(0)
const [retryTick, setRetryTick] = useState(0)
const enqueueScrape = (from: string, to: string) => {
setScrapeQueue(q => q.some(j => j.from === from && j.to === to) ? q : [...q, { from, to }])
@ -969,18 +971,25 @@ const RateMatrixTab: React.FC = () => {
})
return (await api.post('/competitors/scrape', { from_date: job.from, to_date: job.to })).data
},
onError: () => {
onError: (err: any) => {
setScrapeWatch(null)
setScrapeQueue(q => q.slice(1))
if (err?.response?.status === 409) {
// Server is busy with another scrape — keep the job queued, retry in 30s
setDispatchHoldUntil(Date.now() + 30000)
setTimeout(() => setRetryTick(t => t + 1), 31000)
} else {
setScrapeQueue(q => q.slice(1))
}
},
})
// Dispatch the next queued job when idle
useEffect(() => {
if (Date.now() < dispatchHoldUntil) return
if (scrapeQueue.length && !scrapeWatch && !dateScrapeM.isPending) {
dateScrapeM.mutate(scrapeQueue[0])
}
}, [scrapeQueue, scrapeWatch, dateScrapeM.isPending]) // eslint-disable-line react-hooks/exhaustive-deps
}, [scrapeQueue, scrapeWatch, dateScrapeM.isPending, dispatchHoldUntil, retryTick]) // eslint-disable-line react-hooks/exhaustive-deps
// Also serves the date-header Booking.com links via location_name
const { data: watchStatus } = useQuery<ScraperStatus>({