diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index b314056..ad8d3dd 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -524,15 +524,20 @@ def _is_hotel_page_mode(db: Session) -> bool: return (row and row.config_value) == 'playwright_hotel_page' -def _effective_concurrency(db: Session, n_jobs: int) -> int: - """Clamp configured concurrency to the workload, and force serial when the - proxy is off — N workers would share one IP and hammer it, worse than 1.""" +def _effective_concurrency(db: Session, n_jobs: int, require_proxy: bool = True) -> int: + """Clamp configured concurrency to the workload. + + require_proxy=True (search-results mode): force serial without a proxy because + all workers would hit the same search URL from the same IP. + require_proxy=False (hotel-page mode): allow parallelism even without proxy — + each worker hits a different hotel's URL so concurrent requests look like normal + multi-tab browsing rather than a hammered aggregation endpoint. + """ configured = get_scraper_concurrency(db) if configured <= 1 or n_jobs <= 1: return 1 - # get_scraper_backend does no I/O beyond the config read; safe to probe. - if not get_scraper_backend(db)._proxy_enabled(): - logger.info("Proxy disabled — running scrape serially (parallelism needs per-worker IPs)") + if require_proxy and not get_scraper_backend(db)._proxy_enabled(): + logger.info("Proxy disabled — running search-results scrape serially (parallelism needs per-worker IPs)") return 1 return max(1, min(configured, n_jobs)) @@ -767,7 +772,7 @@ async def _run_hotel_page_scrape( ) db.commit() - concurrency = _effective_concurrency(db, len(hotels)) + concurrency = _effective_concurrency(db, len(hotels), require_proxy=False) logger.info( f"Hotel-page scrape {from_date}..{to_date}: " f"{len(hotels)} hotels × {len(dates)} dates = {len(hotel_date_jobs)} jobs, "