From 7b4bb72f1519eefb1f14505dac5ae3e4b2519335 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 17:50:45 +0000 Subject: [PATCH] Add Referer header to search URL navigation A direct hit to searchresults.en-gb.html with no Referer causes Cloudflare to hold the response body open (never sending HTML), so domcontentloaded never fires. Setting referer=HOMEPAGE makes the request look like the user searched from the homepage, which resolves the stall. Co-Authored-By: Claude Sonnet 4.6 --- backend/services/scraper_backends/playwright_local.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 4ed99ea..710e6d2 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -592,7 +592,12 @@ class PlaywrightLocalBackend(ScraperBackend): dest_id=dest_id, location_params=location_params) logger.info(f"Scraping page {page_num + 1}: {url}") try: - await page.goto(url, wait_until='domcontentloaded', timeout=90000) + # referer=HOMEPAGE makes the request look like the user + # searched from the homepage, which is required — a + # direct hit to searchresults with no Referer causes + # Cloudflare to hold the response body open indefinitely. + await page.goto(url, wait_until='domcontentloaded', + timeout=90000, referer=self.HOMEPAGE) except Exception as e: logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False