Market View: queue multiple per-date scrapes, dispatched sequentially
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
349236cd7e
commit
6a3aee4db0
1 changed files with 23 additions and 7 deletions
|
|
@ -922,8 +922,11 @@ const RateMatrixTab: React.FC = () => {
|
|||
const [showDirect, setShowDirect] = useState(false)
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
// The scrape endpoint returns immediately (runs in background), so we watch
|
||||
// scraper status until a new batch finishes, then refetch the matrix.
|
||||
// The scrape endpoint returns immediately (runs in background), and the
|
||||
// scraper handles one date at a time, so queued dates are dispatched
|
||||
// sequentially: watch scraper status until a new batch finishes, refetch
|
||||
// the matrix, then start the next date in the queue.
|
||||
const [scrapeQueue, setScrapeQueue] = useState<string[]>([])
|
||||
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number } | null>(null)
|
||||
|
||||
const dateScrapeM = useMutation({
|
||||
|
|
@ -936,9 +939,17 @@ const RateMatrixTab: React.FC = () => {
|
|||
onError: () => {
|
||||
setScrapingDate(null)
|
||||
setScrapeWatch(null)
|
||||
setScrapeQueue(q => q.slice(1))
|
||||
},
|
||||
})
|
||||
|
||||
// Dispatch the next queued date when idle
|
||||
useEffect(() => {
|
||||
if (scrapeQueue.length && !scrapeWatch && !dateScrapeM.isPending) {
|
||||
dateScrapeM.mutate(scrapeQueue[0])
|
||||
}
|
||||
}, [scrapeQueue, scrapeWatch, dateScrapeM.isPending]) // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
const { data: watchStatus } = useQuery<ScraperStatus>({
|
||||
queryKey: ['scraper-status'],
|
||||
queryFn: async () => (await api.get('/competitors/status')).data,
|
||||
|
|
@ -956,6 +967,7 @@ const RateMatrixTab: React.FC = () => {
|
|||
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
|
||||
setScrapingDate(null)
|
||||
setScrapeWatch(null)
|
||||
setScrapeQueue(q => q.slice(1))
|
||||
}
|
||||
}, [watchStatus, scrapeWatch, queryClient])
|
||||
|
||||
|
|
@ -1155,15 +1167,19 @@ const RateMatrixTab: React.FC = () => {
|
|||
<span style={styles.scrapeAge}>{scrapeAge}</span>
|
||||
) : null}
|
||||
<button
|
||||
onClick={() => dateScrapeM.mutate(d)}
|
||||
disabled={scrapingDate !== null}
|
||||
onClick={() => setScrapeQueue(q => q.includes(d) ? q : [...q, d])}
|
||||
disabled={scrapeQueue.includes(d)}
|
||||
style={mergeStyles(
|
||||
styles.scrapeBtn,
|
||||
scrapingDate === d ? styles.scrapeBtnActive : {}
|
||||
scrapeQueue.includes(d) ? styles.scrapeBtnActive : {}
|
||||
)}
|
||||
title={`Scrape ${d}`}
|
||||
title={
|
||||
scrapingDate === d ? `Scraping ${d}…`
|
||||
: scrapeQueue.includes(d) ? `Queued (#${scrapeQueue.indexOf(d) + 1})`
|
||||
: `Scrape ${d}`
|
||||
}
|
||||
>
|
||||
{scrapingDate === d ? '...' : '↻'}
|
||||
{scrapingDate === d ? '...' : scrapeQueue.includes(d) ? `#${scrapeQueue.indexOf(d) + 1}` : '↻'}
|
||||
</button>
|
||||
</div>
|
||||
</th>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue