Parallel scrape workers (proxy-gated, configurable)
Fan a scrape run out across N workers, each with its own DB session and its own scraper backend — and since every backend picks a random sticky session id, each worker scrapes from a distinct residential IP. Dates are interleaved across workers so each covers a spread of the range. Cuts a 150-date batch from ~30-45 min to ~12-15 min at 3 workers. - booking_scraper_concurrency config key (default 3); Settings → Scraper Proxy has a "Parallel workers" field - Forced to 1 when the proxy is off (N workers would share one IP and hammer it) or when there's a single date - Both manual and queue paths routed through _scrape_dates_concurrent; workers mark their own queue items - Per-worker rotate-on-block replaces the old global pause-on-block Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
579a189cb8
commit
6b7f00b40a
2 changed files with 204 additions and 135 deletions
|
|
@ -123,6 +123,12 @@ function ProxyTab() {
|
|||
const [password, setPassword] = useState('')
|
||||
const [testResult, setTestResult] = useState<ProxyTestResult | null>(null)
|
||||
const [testError, setTestError] = useState<string | null>(null)
|
||||
const [workers, setWorkers] = useState('3')
|
||||
|
||||
const { data: sysCfg } = useQuery<SystemConfig>({
|
||||
queryKey: ['system-config'],
|
||||
queryFn: () => api.get('/competitors/config/system').then(r => r.data),
|
||||
})
|
||||
|
||||
// Seed form once the stored config arrives (password stays blank by design)
|
||||
useEffect(() => {
|
||||
|
|
@ -135,6 +141,17 @@ function ProxyTab() {
|
|||
}
|
||||
}, [cfg])
|
||||
|
||||
useEffect(() => {
|
||||
if (sysCfg?.booking_scraper_concurrency) setWorkers(String(sysCfg.booking_scraper_concurrency))
|
||||
}, [sysCfg])
|
||||
|
||||
const saveWorkers = useMutation({
|
||||
mutationFn: () => api.post('/competitors/config/system', {
|
||||
key: 'booking_scraper_concurrency', value: String(Math.max(1, parseInt(workers) || 1)),
|
||||
}),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ['system-config'] }),
|
||||
})
|
||||
|
||||
const save = useMutation({
|
||||
mutationFn: () =>
|
||||
api.post('/competitors/config/proxy', {
|
||||
|
|
@ -248,6 +265,38 @@ function ProxyTab() {
|
|||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="card">
|
||||
<div className="card-header">
|
||||
<span style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<Network size={16} strokeWidth={1.75} />
|
||||
Parallel Workers
|
||||
</span>
|
||||
</div>
|
||||
<div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 12 }}>
|
||||
<p style={{ fontSize: 13, color: 'var(--text-mid)', margin: 0 }}>
|
||||
How many dates to scrape at once. Each worker runs its own browser on its own
|
||||
residential IP, so a run finishes roughly this many times faster. Only takes effect
|
||||
when the proxy is enabled (without it, all workers would share one IP). Budget about
|
||||
0.4 GB of RAM per worker — 3 is safe on the current 4 GB; raise it after
|
||||
adding RAM.
|
||||
</p>
|
||||
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
|
||||
<input type="number" min={1} max={12} value={workers}
|
||||
onChange={e => setWorkers(e.target.value)} style={{ width: 90 }} />
|
||||
<button className="btn btn-primary btn-sm" onClick={() => saveWorkers.mutate()}
|
||||
disabled={saveWorkers.isPending}>
|
||||
<Save size={13} strokeWidth={1.75} />
|
||||
{saveWorkers.isPending ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
{!enabled && (
|
||||
<span style={{ fontSize: 12, color: 'var(--warning)' }}>
|
||||
Enable the proxy above for this to apply
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -493,6 +542,7 @@ function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; is
|
|||
'booking_proxy_enabled',
|
||||
'booking_proxy_host',
|
||||
'booking_proxy_country',
|
||||
'booking_scraper_concurrency',
|
||||
'sync_newbook_current_rates_enabled',
|
||||
'sync_newbook_current_rates_time',
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue