Market View: Scrape 7d/30d buttons, Booking.com day-search links on date headers
- Scrape queue generalised to date-range jobs; range and per-date scrapes share one sequential queue (timeout scales with range length) - Matrix refetches every 30s while a scrape runs so columns fill in live - Eye icon per date header opens that night's location search on Booking.com Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
6a3aee4db0
commit
77284b3ecc
1 changed files with 95 additions and 29 deletions
|
|
@ -1,14 +1,35 @@
|
|||
import React, { useState, useMemo, useCallback, useEffect } from 'react'
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { Eye } from 'lucide-react'
|
||||
import api from '../api'
|
||||
|
||||
// Format Date as YYYY-MM-DD using local time (avoids UTC/DST shift from toISOString)
|
||||
const fmtDate = (d: Date) => `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
|
||||
|
||||
// Booking.com search results for the scrape location on a given night
|
||||
const buildSearchUrl = (location: string, checkin: string) => {
|
||||
const co = new Date(checkin + 'T00:00:00')
|
||||
co.setDate(co.getDate() + 1)
|
||||
const params = new URLSearchParams({
|
||||
ss: location,
|
||||
checkin,
|
||||
checkout: fmtDate(co),
|
||||
group_adults: '2',
|
||||
no_rooms: '1',
|
||||
group_children: '0',
|
||||
})
|
||||
return `https://www.booking.com/searchresults.html?${params}`
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// TYPES
|
||||
// ============================================
|
||||
|
||||
interface ScrapeJob {
|
||||
from: string
|
||||
to: string
|
||||
}
|
||||
|
||||
interface ScraperStatus {
|
||||
enabled: boolean
|
||||
paused: boolean
|
||||
|
|
@ -915,7 +936,6 @@ const RateMatrixTab: React.FC = () => {
|
|||
})
|
||||
const [includeMarket, setIncludeMarket] = useState(false)
|
||||
const [hoveredCell, setHoveredCell] = useState<{ row: number; col: number } | null>(null)
|
||||
const [scrapingDate, setScrapingDate] = useState<string | null>(null)
|
||||
const [rangeMode, setRangeMode] = useState(false)
|
||||
const [customFrom, setCustomFrom] = useState(fmtDate(new Date()))
|
||||
const [customTo, setCustomTo] = useState(fmtDate(new Date(Date.now() + 13 * 86400000)))
|
||||
|
|
@ -923,49 +943,61 @@ const RateMatrixTab: React.FC = () => {
|
|||
const queryClient = useQueryClient()
|
||||
|
||||
// The scrape endpoint returns immediately (runs in background), and the
|
||||
// scraper handles one date at a time, so queued dates are dispatched
|
||||
// sequentially: watch scraper status until a new batch finishes, refetch
|
||||
// the matrix, then start the next date in the queue.
|
||||
const [scrapeQueue, setScrapeQueue] = useState<string[]>([])
|
||||
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number } | null>(null)
|
||||
// scraper handles one job at a time, so queued jobs (single dates or
|
||||
// ranges) are dispatched sequentially: watch scraper status until a new
|
||||
// batch finishes, refetch the matrix, then start the next job.
|
||||
const [scrapeQueue, setScrapeQueue] = useState<ScrapeJob[]>([])
|
||||
const [scrapeWatch, setScrapeWatch] = useState<{ prevBatch: string | null, startedAt: number, timeoutMs: number } | null>(null)
|
||||
|
||||
const enqueueScrape = (from: string, to: string) => {
|
||||
setScrapeQueue(q => q.some(j => j.from === from && j.to === to) ? q : [...q, { from, to }])
|
||||
}
|
||||
|
||||
const activeJob = scrapeWatch ? scrapeQueue[0] : undefined
|
||||
const scrapingDate = activeJob && activeJob.from === activeJob.to ? activeJob.from : null
|
||||
|
||||
const dateScrapeM = useMutation({
|
||||
mutationFn: async (d: string) => {
|
||||
mutationFn: async (job: ScrapeJob) => {
|
||||
const st = (await api.get('/competitors/status')).data as ScraperStatus
|
||||
setScrapingDate(d)
|
||||
setScrapeWatch({ prevBatch: st?.last_scrape?.batch_id ?? null, startedAt: Date.now() })
|
||||
return (await api.post('/competitors/scrape', { from_date: d, to_date: d })).data
|
||||
const days = Math.round((new Date(job.to).getTime() - new Date(job.from).getTime()) / 86400000) + 1
|
||||
setScrapeWatch({
|
||||
prevBatch: st?.last_scrape?.batch_id ?? null,
|
||||
startedAt: Date.now(),
|
||||
// generous per-date allowance: ranges take ~1 min per date
|
||||
timeoutMs: 10 * 60 * 1000 + days * 90 * 1000,
|
||||
})
|
||||
return (await api.post('/competitors/scrape', { from_date: job.from, to_date: job.to })).data
|
||||
},
|
||||
onError: () => {
|
||||
setScrapingDate(null)
|
||||
setScrapeWatch(null)
|
||||
setScrapeQueue(q => q.slice(1))
|
||||
},
|
||||
})
|
||||
|
||||
// Dispatch the next queued date when idle
|
||||
// Dispatch the next queued job when idle
|
||||
useEffect(() => {
|
||||
if (scrapeQueue.length && !scrapeWatch && !dateScrapeM.isPending) {
|
||||
dateScrapeM.mutate(scrapeQueue[0])
|
||||
}
|
||||
}, [scrapeQueue, scrapeWatch, dateScrapeM.isPending]) // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
// Also serves the date-header Booking.com links via location_name
|
||||
const { data: watchStatus } = useQuery<ScraperStatus>({
|
||||
queryKey: ['scraper-status'],
|
||||
queryFn: async () => (await api.get('/competitors/status')).data,
|
||||
enabled: scrapeWatch !== null,
|
||||
refetchInterval: scrapeWatch !== null ? 5000 : false,
|
||||
staleTime: 60 * 1000,
|
||||
})
|
||||
const locationName = watchStatus?.location_name
|
||||
|
||||
useEffect(() => {
|
||||
if (!scrapeWatch) return
|
||||
const ls = watchStatus?.last_scrape
|
||||
const finished = ls && ls.batch_id !== scrapeWatch.prevBatch && ls.status !== 'running'
|
||||
const timedOut = Date.now() - scrapeWatch.startedAt > 10 * 60 * 1000
|
||||
const timedOut = Date.now() - scrapeWatch.startedAt > scrapeWatch.timeoutMs
|
||||
if (finished || timedOut) {
|
||||
queryClient.invalidateQueries({ queryKey: ['competitor-matrix'] })
|
||||
queryClient.invalidateQueries({ queryKey: ['scraper-status'] })
|
||||
setScrapingDate(null)
|
||||
setScrapeWatch(null)
|
||||
setScrapeQueue(q => q.slice(1))
|
||||
}
|
||||
|
|
@ -995,6 +1027,8 @@ const RateMatrixTab: React.FC = () => {
|
|||
})
|
||||
return (await api.get(`/competitors/matrix?${params}`)).data
|
||||
},
|
||||
// While a scrape runs, refresh so columns fill in as dates complete
|
||||
refetchInterval: scrapeWatch !== null ? 30000 : false,
|
||||
})
|
||||
|
||||
const dates = data?.dates || []
|
||||
|
|
@ -1115,6 +1149,24 @@ const RateMatrixTab: React.FC = () => {
|
|||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||
{[{ label: 'Scrape 7d', days: 7 }, { label: 'Scrape 30d', days: 30 }].map(p => {
|
||||
const from = fmtDate(new Date())
|
||||
const to = fmtDate(new Date(Date.now() + (p.days - 1) * 86400000))
|
||||
const queued = scrapeQueue.some(j => j.from === from && j.to === to)
|
||||
const active = activeJob && activeJob.from === from && activeJob.to === to
|
||||
return (
|
||||
<button key={p.label}
|
||||
style={buttonStyle('outline', 'small')}
|
||||
disabled={queued}
|
||||
onClick={() => enqueueScrape(from, to)}
|
||||
title={`Scrape today + next ${p.days - 1} days (~${p.days} min)`}
|
||||
>
|
||||
{active ? 'Scraping…' : queued ? 'Queued' : p.label}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
<label style={styles.checkboxLabel}>
|
||||
<input
|
||||
type="checkbox"
|
||||
|
|
@ -1149,6 +1201,7 @@ const RateMatrixTab: React.FC = () => {
|
|||
{dates.map((d, colIdx) => {
|
||||
const scrapeAge = formatScrapeAge(scrapedAtByDate[d])
|
||||
const isColHovered = hoveredCell?.col === colIdx
|
||||
const queuePos = scrapeQueue.findIndex(j => j.from === d && j.to === d)
|
||||
return (
|
||||
<th
|
||||
key={d}
|
||||
|
|
@ -1166,21 +1219,34 @@ const RateMatrixTab: React.FC = () => {
|
|||
{scrapeAge ? (
|
||||
<span style={styles.scrapeAge}>{scrapeAge}</span>
|
||||
) : null}
|
||||
<button
|
||||
onClick={() => setScrapeQueue(q => q.includes(d) ? q : [...q, d])}
|
||||
disabled={scrapeQueue.includes(d)}
|
||||
style={mergeStyles(
|
||||
styles.scrapeBtn,
|
||||
scrapeQueue.includes(d) ? styles.scrapeBtnActive : {}
|
||||
<div style={{ display: 'flex', gap: 2, alignItems: 'center', justifyContent: 'center' }}>
|
||||
<button
|
||||
onClick={() => enqueueScrape(d, d)}
|
||||
disabled={queuePos >= 0}
|
||||
style={mergeStyles(
|
||||
styles.scrapeBtn,
|
||||
queuePos >= 0 ? styles.scrapeBtnActive : {}
|
||||
)}
|
||||
title={
|
||||
scrapingDate === d ? `Scraping ${d}…`
|
||||
: queuePos >= 0 ? `Queued (#${queuePos + 1})`
|
||||
: `Scrape ${d}`
|
||||
}
|
||||
>
|
||||
{scrapingDate === d ? '...' : queuePos >= 0 ? `#${queuePos + 1}` : '↻'}
|
||||
</button>
|
||||
{locationName && (
|
||||
<a
|
||||
href={buildSearchUrl(locationName, d)}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
title={`View ${d} search on Booking.com`}
|
||||
style={{ color: 'var(--text-mid)', display: 'inline-flex', alignItems: 'center' }}
|
||||
>
|
||||
<Eye size={11} strokeWidth={1.75} />
|
||||
</a>
|
||||
)}
|
||||
title={
|
||||
scrapingDate === d ? `Scraping ${d}…`
|
||||
: scrapeQueue.includes(d) ? `Queued (#${scrapeQueue.indexOf(d) + 1})`
|
||||
: `Scrape ${d}`
|
||||
}
|
||||
>
|
||||
{scrapingDate === d ? '...' : scrapeQueue.includes(d) ? `#${scrapeQueue.indexOf(d) + 1}` : '↻'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</th>
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue