Market View: queue multiple per-date scrapes, dispatched sequentially
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
349236cd7e
commit
6a3aee4db0
1 changed files with 23 additions and 7 deletions
|
|
@ -922,8 +922,11 @@ const RateMatrixTab: React.FC = () => {
|
||||||
const [showDirect, setShowDirect] = useState(false)
|
const [showDirect, setShowDirect] = useState(false)
|
||||||
const queryClient = useQueryClient()
|
const queryClient = useQueryClient()
|
||||||
|
|
||||||
// The scrape endpoint returns immediately (runs in background), so we watch
|
// The scrape endpoint returns immediately (runs in background), and the
|
||||||
// scraper status until a new batch finishes, then refetch the matrix.
|
// scraper handles one date at a time, so queued dates are dispatched
|
||||||
|
// sequentially: watch scraper status until a new batch finishes, refetch
|
||||||
|
// the matrix, then start the next date in the queue.
|
||||||
|
const [scrapeQueue, setScrapeQueue] = useState<string[]>([])
|
||||||
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number } | null>(null)
|
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number } | null>(null)
|
||||||
|
|
||||||
const dateScrapeM = useMutation({
|
const dateScrapeM = useMutation({
|
||||||
|
|
@ -936,9 +939,17 @@ const RateMatrixTab: React.FC = () => {
|
||||||
onError: () => {
|
onError: () => {
|
||||||
setScrapingDate(null)
|
setScrapingDate(null)
|
||||||
setScrapeWatch(null)
|
setScrapeWatch(null)
|
||||||
|
setScrapeQueue(q => q.slice(1))
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Dispatch the next queued date when idle
|
||||||
|
useEffect(() => {
|
||||||
|
if (scrapeQueue.length && !scrapeWatch && !dateScrapeM.isPending) {
|
||||||
|
dateScrapeM.mutate(scrapeQueue[0])
|
||||||
|
}
|
||||||
|
}, [scrapeQueue, scrapeWatch, dateScrapeM.isPending]) // eslint-disable-line react-hooks/exhaustive-deps
|
||||||
|
|
||||||
const { data: watchStatus } = useQuery<ScraperStatus>({
|
const { data: watchStatus } = useQuery<ScraperStatus>({
|
||||||
queryKey: ['scraper-status'],
|
queryKey: ['scraper-status'],
|
||||||
queryFn: async () => (await api.get('/competitors/status')).data,
|
queryFn: async () => (await api.get('/competitors/status')).data,
|
||||||
|
|
@ -956,6 +967,7 @@ const RateMatrixTab: React.FC = () => {
|
||||||
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
|
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
|
||||||
setScrapingDate(null)
|
setScrapingDate(null)
|
||||||
setScrapeWatch(null)
|
setScrapeWatch(null)
|
||||||
|
setScrapeQueue(q => q.slice(1))
|
||||||
}
|
}
|
||||||
}, [watchStatus, scrapeWatch, queryClient])
|
}, [watchStatus, scrapeWatch, queryClient])
|
||||||
|
|
||||||
|
|
@ -1155,15 +1167,19 @@ const RateMatrixTab: React.FC = () => {
|
||||||
<span style={styles.scrapeAge}>{scrapeAge}</span>
|
<span style={styles.scrapeAge}>{scrapeAge}</span>
|
||||||
) : null}
|
) : null}
|
||||||
<button
|
<button
|
||||||
onClick={() => dateScrapeM.mutate(d)}
|
onClick={() => setScrapeQueue(q => q.includes(d) ? q : [...q, d])}
|
||||||
disabled={scrapingDate !== null}
|
disabled={scrapeQueue.includes(d)}
|
||||||
style={mergeStyles(
|
style={mergeStyles(
|
||||||
styles.scrapeBtn,
|
styles.scrapeBtn,
|
||||||
scrapingDate === d ? styles.scrapeBtnActive : {}
|
scrapeQueue.includes(d) ? styles.scrapeBtnActive : {}
|
||||||
)}
|
)}
|
||||||
title={`Scrape ${d}`}
|
title={
|
||||||
|
scrapingDate === d ? `Scraping ${d}…`
|
||||||
|
: scrapeQueue.includes(d) ? `Queued (#${scrapeQueue.indexOf(d) + 1})`
|
||||||
|
: `Scrape ${d}`
|
||||||
|
}
|
||||||
>
|
>
|
||||||
{scrapingDate === d ? '...' : '↻'}
|
{scrapingDate === d ? '...' : scrapeQueue.includes(d) ? `#${scrapeQueue.indexOf(d) + 1}` : '↻'}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</th>
|
</th>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue