Market View: queue multiple per-date scrapes, dispatched sequentially

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 15:09:29 +00:00
parent 349236cd7e
commit 6a3aee4db0

View file

@ -922,8 +922,11 @@ const RateMatrixTab: React.FC = () => {
const [showDirect, setShowDirect] = useState(false) const [showDirect, setShowDirect] = useState(false)
const queryClient = useQueryClient() const queryClient = useQueryClient()
// The scrape endpoint returns immediately (runs in background), so we watch // The scrape endpoint returns immediately (runs in background), and the
// scraper status until a new batch finishes, then refetch the matrix. // scraper handles one date at a time, so queued dates are dispatched
// sequentially: watch scraper status until a new batch finishes, refetch
// the matrix, then start the next date in the queue.
const [scrapeQueue, setScrapeQueue] = useState<string[]>([])
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number } | null>(null) const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number } | null>(null)
const dateScrapeM = useMutation({ const dateScrapeM = useMutation({
@ -936,9 +939,17 @@ const RateMatrixTab: React.FC = () => {
onError: () => { onError: () => {
setScrapingDate(null) setScrapingDate(null)
setScrapeWatch(null) setScrapeWatch(null)
setScrapeQueue(q => q.slice(1))
}, },
}) })
// Dispatch the next queued date when idle
useEffect(() => {
if (scrapeQueue.length && !scrapeWatch && !dateScrapeM.isPending) {
dateScrapeM.mutate(scrapeQueue[0])
}
}, [scrapeQueue, scrapeWatch, dateScrapeM.isPending]) // eslint-disable-line react-hooks/exhaustive-deps
const { data: watchStatus } = useQuery<ScraperStatus>({ const { data: watchStatus } = useQuery<ScraperStatus>({
queryKey: ['scraper-status'], queryKey: ['scraper-status'],
queryFn: async () => (await api.get('/competitors/status')).data, queryFn: async () => (await api.get('/competitors/status')).data,
@ -956,6 +967,7 @@ const RateMatrixTab: React.FC = () => {
queryClient.invalidateQueries({ queryKey: ['scraper-status'] }) queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
setScrapingDate(null) setScrapingDate(null)
setScrapeWatch(null) setScrapeWatch(null)
setScrapeQueue(q => q.slice(1))
} }
}, [watchStatus, scrapeWatch, queryClient]) }, [watchStatus, scrapeWatch, queryClient])
@ -1155,15 +1167,19 @@ const RateMatrixTab: React.FC = () => {
<span style={styles.scrapeAge}>{scrapeAge}</span> <span style={styles.scrapeAge}>{scrapeAge}</span>
) : null} ) : null}
<button <button
onClick={() => dateScrapeM.mutate(d)} onClick={() => setScrapeQueue(q => q.includes(d) ? q : [...q, d])}
disabled={scrapingDate !== null} disabled={scrapeQueue.includes(d)}
style={mergeStyles( style={mergeStyles(
styles.scrapeBtn, styles.scrapeBtn,
scrapingDate === d ? styles.scrapeBtnActive : {} scrapeQueue.includes(d) ? styles.scrapeBtnActive : {}
)} )}
title={`Scrape ${d}`} title={
scrapingDate === d ? `Scraping ${d}`
: scrapeQueue.includes(d) ? `Queued (#${scrapeQueue.indexOf(d) + 1})`
: `Scrape ${d}`
}
> >
{scrapingDate === d ? '...' : '↻'} {scrapingDate === d ? '...' : scrapeQueue.includes(d) ? `#${scrapeQueue.indexOf(d) + 1}` : '↻'}
</button> </button>
</div> </div>
</th> </th>