diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 4ed99ea..710e6d2 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -592,7 +592,12 @@ class PlaywrightLocalBackend(ScraperBackend): dest_id=dest_id, location_params=location_params) logger.info(f"Scraping page {page_num + 1}: {url}") try: - await page.goto(url, wait_until='domcontentloaded', timeout=90000) + # referer=HOMEPAGE makes the request look like the user + # searched from the homepage, which is required — a + # direct hit to searchresults with no Referer causes + # Cloudflare to hold the response body open indefinitely. + await page.goto(url, wait_until='domcontentloaded', + timeout=90000, referer=self.HOMEPAGE) except Exception as e: logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False