Parallel scrape workers (proxy-gated, configurable)

Fan a scrape run out across N workers, each with its own DB session and
its own scraper backend — and since every backend picks a random sticky
session id, each worker scrapes from a distinct residential IP. Dates are
interleaved across workers so each covers a spread of the range. Cuts a
150-date batch from ~30-45 min to ~12-15 min at 3 workers.

- booking_scraper_concurrency config key (default 3); Settings → Scraper
  Proxy has a "Parallel workers" field
- Forced to 1 when the proxy is off (N workers would share one IP and
  hammer it) or when there's a single date
- Both manual and queue paths routed through _scrape_dates_concurrent;
  workers mark their own queue items
- Per-worker rotate-on-block replaces the old global pause-on-block

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 22:13:23 +00:00
parent 579a189cb8
commit 6b7f00b40a
2 changed files with 204 additions and 135 deletions

View file

@ -123,6 +123,12 @@ function ProxyTab() {
const [password, setPassword] = useState('')
const [testResult, setTestResult] = useState<ProxyTestResult | null>(null)
const [testError, setTestError] = useState<string | null>(null)
const [workers, setWorkers] = useState('3')
const { data: sysCfg } = useQuery<SystemConfig>({
queryKey: ['system-config'],
queryFn: () => api.get('/competitors/config/system').then(r => r.data),
})
// Seed form once the stored config arrives (password stays blank by design)
useEffect(() => {
@ -135,6 +141,17 @@ function ProxyTab() {
}
}, [cfg])
useEffect(() => {
if (sysCfg?.booking_scraper_concurrency) setWorkers(String(sysCfg.booking_scraper_concurrency))
}, [sysCfg])
const saveWorkers = useMutation({
mutationFn: () => api.post('/competitors/config/system', {
key: 'booking_scraper_concurrency', value: String(Math.max(1, parseInt(workers) || 1)),
}),
onSuccess: () => qc.invalidateQueries({ queryKey: ['system-config'] }),
})
const save = useMutation({
mutationFn: () =>
api.post('/competitors/config/proxy', {
@ -248,6 +265,38 @@ function ProxyTab() {
)}
</div>
</div>
<div className="card">
<div className="card-header">
<span style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<Network size={16} strokeWidth={1.75} />
Parallel Workers
</span>
</div>
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 12 }}>
<p style={{ fontSize: 13, color: 'var(--text-mid)', margin: 0 }}>
How many dates to scrape at once. Each worker runs its own browser on its own
residential IP, so a run finishes roughly this many times faster. Only takes effect
when the proxy is enabled (without it, all workers would share one IP). Budget about
0.4&nbsp;GB of RAM per worker 3 is safe on the current 4&nbsp;GB; raise it after
adding RAM.
</p>
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
<input type="number" min={1} max={12} value={workers}
onChange={e => setWorkers(e.target.value)} style={{ width: 90 }} />
<button className="btn btn-primary btn-sm" onClick={() => saveWorkers.mutate()}
disabled={saveWorkers.isPending}>
<Save size={13} strokeWidth={1.75} />
{saveWorkers.isPending ? 'Saving…' : 'Save'}
</button>
{!enabled && (
<span style={{ fontSize: 12, color: 'var(--warning)' }}>
Enable the proxy above for this to apply
</span>
)}
</div>
</div>
</div>
</div>
)
}
@ -493,6 +542,7 @@ function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; is
'booking_proxy_enabled',
'booking_proxy_host',
'booking_proxy_country',
'booking_scraper_concurrency',
'sync_newbook_current_rates_enabled',
'sync_newbook_current_rates_time',
]