hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Reduce scraper timeouts and raise default concurrency to 6

Browse source 
Timeouts were set for the old proxy 407 round-trip world (14.5s latency
per challenge). Now that credentials are embedded in the proxy URL, pages
load in ~7s — the 90s goto timeout was forcing 5+ minutes of wasted wait
on Cloudflare-blocked dates before rotating sessions.

  homepage warmup goto: 90s → 20s
  page 1 search goto:   90s → 30s  (4× normal load time headroom)
  property-card waits:  30s → 15s  (cards appear in <3s on clean pages)

Default concurrency 3 → 6: each worker uses its own residential proxy IP
so parallelism is safe. ~0.4 GB per Chromium; 6 workers fits in 4 GB LXC.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-09 21:17:19 +00:00
parent b47258d47b
commit ea66025140
2 changed files with 9 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

10
backend/services/booking_scraper.py
View file

@ -426,16 +426,16 @@ async def scrape_date(
def get_scraper_concurrency(db: Session) -> int:
"""Number of parallel scrape workers (config key, default 3). Each worker
runs its own browser on its own residential proxy IP, so raise this only
with proxy IPs and RAM to spare (~0.4 GB per worker)."""
"""Number of parallel scrape workers (config key, default 6). Each worker
runs its own browser on its own residential proxy IP (~0.4 GB each), so
keep this below RAM / 0.4 GB. Requires proxy — serial fallback if off."""
row = db.execute(
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_concurrency'")
).fetchone()
try:
return max(1, int(row.config_value)) if row and row.config_value else 3
return max(1, int(row.config_value)) if row and row.config_value else 6
except (ValueError, TypeError):
return 3
return 6
def _effective_concurrency(db: Session, n_jobs: int) -> int:

8
backend/services/scraper_backends/playwright_local.py
View file

@ -243,7 +243,7 @@ class PlaywrightLocalBackend(ScraperBackend):
async def _warm_up(self, page: Page):
"""Visit the homepage first so search requests carry session cookies."""
try:
await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=90000)
await page.goto(self.HOMEPAGE, wait_until='domcontentloaded', timeout=20000)
await asyncio.sleep(random.uniform(1.5, 3.0))
# Dismiss the cookie-consent dialog if present
for sel in ['#onetrust-accept-btn-handler',
@ -375,12 +375,12 @@ class PlaywrightLocalBackend(ScraperBackend):
# Wait for property cards - booking.com uses data-testid. Retry once
# with a scroll nudge before giving up: cards can lazy-load late.
try:
await page.wait_for_selector('[data-testid="property-card"]', timeout=30000)
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
except Exception as e:
logger.warning(f"No property cards on first wait, retrying: {e}")
try:
await self._human_like_scroll(page)
await page.wait_for_selector('[data-testid="property-card"]', timeout=30000)
await page.wait_for_selector('[data-testid="property-card"]', timeout=15000)
except Exception as e2:
logger.warning(f"No property cards found after retry: {e2}")
return hotels, rates
@ -614,7 +614,7 @@ class PlaywrightLocalBackend(ScraperBackend):
# direct hit to searchresults with no Referer causes
# Cloudflare to hold the response body open indefinitely.
await page.goto(url, wait_until='domcontentloaded',
timeout=90000, referer=self.HOMEPAGE)
timeout=30000, referer=self.HOMEPAGE)
except Exception as e:
logger.warning(f"Page load timeout, continuing: {e}")
page_loaded = False