From 8ccc0e272d96c2fc35a6b83eaf5671e4e7c678d7 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 10 Jul 2026 01:22:51 +0000 Subject: [PATCH] Fix max_persons extraction: read from span text not block_id segment 2 Old Stocks block_id has 0 in position 2 (not the persons count) so all its rates were stored as max_persons=0 and deprioritised in the matrix. Now reads max_persons from the visible 'Max persons: N' span (matching the original scrapy spider), with aria-label/title fallback. block_id segment 2 is unreliable across different hotel layouts. Also treat max_persons=0 as unknown in the matrix ORDER BY priority. Co-Authored-By: Claude Sonnet 4.6 --- backend/api/competitors.py | 4 ++-- .../scraper_backends/playwright_hotel_page.py | 22 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/api/competitors.py b/backend/api/competitors.py index b441f9b..02504fd 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -779,7 +779,7 @@ async def get_competitor_matrix( AND h.is_active = TRUE AND r.rate_date >= :from_date AND r.rate_date <= :to_date ORDER BY r.hotel_id, r.rate_date, - CASE WHEN r.max_persons IS NULL OR r.max_persons = 2 THEN 0 ELSE 1 END, + CASE WHEN r.max_persons IS NULL OR r.max_persons = 0 OR r.max_persons = 2 THEN 0 ELSE 1 END, r.scraped_at DESC, r.rate_gross ASC NULLS LAST """), {'from_date': start, 'to_date': end} @@ -820,7 +820,7 @@ async def get_competitor_matrix( AND r.availability_status = 'available' AND r.rate_gross IS NOT NULL ORDER BY r.hotel_id, r.rate_date, - CASE WHEN r.max_persons IS NULL OR r.max_persons = 2 THEN 0 ELSE 1 END, + CASE WHEN r.max_persons IS NULL OR r.max_persons = 0 OR r.max_persons = 2 THEN 0 ELSE 1 END, r.scraped_at DESC, r.rate_gross ASC NULLS LAST """), {'from_date': start, 'to_date': end} diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py index ec9d760..387ccbb 100644 --- a/backend/services/scraper_backends/playwright_hotel_page.py +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -84,10 +84,9 @@ _EXTRACT_RATES_JS = """ if (!blockId) continue; const blockParts = blockId.split('_'); - if (blockParts.length < 3) continue; + if (blockParts.length < 2) continue; - const roomId = blockParts[0]; - const maxPersons = parseInt(blockParts[2]); + const roomId = blockParts[0]; // Room name: the first row for each room type carries the room_type_id_ element. // Subsequent rows for the same room type don't — reuse stored name (same as scrapy). @@ -118,6 +117,21 @@ _EXTRACT_RATES_JS = """ const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i); const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null; + // Max persons: read from visible span text (same as original scrapy spider). + // block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy. + let maxPersons = null; + for (const span of tr.querySelectorAll('span')) { + const t = span.innerText || ''; + const m = t.match(/Max persons?:?\\s*(\\d+)/i); + if (m) { maxPersons = parseInt(m[1]); break; } + } + // Also try occupancy icon count (aria-label or title with "X adults") + if (maxPersons === null) { + const occMatch = tr.innerHTML.match(/title="(\\d+) adults?"/i) + || tr.innerHTML.match(/aria-label="(\\d+) adults?"/i); + if (occMatch) maxPersons = parseInt(occMatch[1]); + } + // Scarcity text ("We have 2 left") as fallback for availability const availText = tr.querySelector('.only_x_left, .thisRoomAvailabilityNew span') ?.innerText?.trim() || ''; @@ -151,7 +165,7 @@ _EXTRACT_RATES_JS = """ breakfast_included: breakfastIncluded, non_refundable: nonRefundable, free_cancel_text: freeCancelText, - max_persons: !isNaN(maxPersons) ? maxPersons : null, + max_persons: maxPersons, }); }