diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index d83d3a9..3a070a0 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -426,16 +426,16 @@ async def scrape_date( def get_scraper_concurrency(db: Session) -> int: - """Number of parallel scrape workers (config key, default 3). Each worker - runs its own browser on its own residential proxy IP, so raise this only - with proxy IPs and RAM to spare (~0.4 GB per worker).""" + """Number of parallel scrape workers (config key, default 6). Each worker + runs its own browser on its own residential proxy IP (~0.4 GB each), so + keep this below RAM / 0.4 GB. Requires proxy — serial fallback if off.""" row = db.execute( text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_concurrency'") ).fetchone() try: - return max(1, int(row.config_value)) if row and row.config_value else 3 + return max(1, int(row.config_value)) if row and row.config_value else 6 except (ValueError, TypeError): - return 3 + return 6 def _effective_concurrency(db: Session, n_jobs: int) -> int: diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 698f4c4..0d0ff1a 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -243,7 +243,7 @@ class PlaywrightLocalBackend(ScraperBackend): async def _warm_up(self, page: Page): """Visit the homepage first so search requests carry session cookies.""" try: - await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=90000) + await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=20000) await asyncio.sleep(random.uniform(1.5, 3.0)) # Dismiss the cookie-consent dialog if present for sel in ['#onetrust-accept-btn-handler', @@ -375,12 +375,12 @@ class PlaywrightLocalBackend(ScraperBackend): # Wait for property cards - booking.com uses data-testid. Retry once # with a scroll nudge before giving up: cards can lazy-load late. try: - await page.wait_for_selector('[data-testid="property-card"]', timeout=30000) + await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) except Exception as e: logger.warning(f"No property cards on first wait, retrying: {e}") try: await self._human_like_scroll(page) - await page.wait_for_selector('[data-testid="property-card"]', timeout=30000) + await page.wait_for_selector('[data-testid="property-card"]', timeout=15000) except Exception as e2: logger.warning(f"No property cards found after retry: {e2}") return hotels, rates @@ -614,7 +614,7 @@ class PlaywrightLocalBackend(ScraperBackend): # direct hit to searchresults with no Referer causes # Cloudflare to hold the response body open indefinitely. await page.goto(url, wait_until='domcontentloaded', - timeout=90000, referer=self.HOMEPAGE) + timeout=30000, referer=self.HOMEPAGE) except Exception as e: logger.warning(f"Page load timeout, continuing: {e}") page_loaded = False