From 83b46bee8fe83e26776d2b122e1214cdb5aacfec Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 10 Jul 2026 00:37:35 +0000 Subject: [PATCH] Fix JS extractor under-counting rooms: iterate all tbody rows like scrapy The previous approach walked siblings from [id^="room_type_id_"] anchors and only processed rows with js-rt-block-row class, causing the first rate row of some room types to be silently skipped when that class was absent. Now iterates all #available_rooms tbody tr rows with data-block-id (same strategy as the original scrapy spider), using the room_type_id_ element presence to identify room names only on the first row, with fallback to the stored name for subsequent rows of the same room type. Co-Authored-By: Claude Sonnet 4.6 --- .../scraper_backends/playwright_hotel_page.py | 154 +++++++++--------- 1 file changed, 78 insertions(+), 76 deletions(-) diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py index 1c94247..c6b24b2 100644 --- a/backend/services/scraper_backends/playwright_hotel_page.py +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -36,93 +36,95 @@ logger = logging.getLogger(__name__) # JS that extracts all rate plan rows from the room availability table. # Runs inside the page after the room table has loaded. +# +# Iterates ALL tbody rows (matching the original scrapy spider approach) rather than +# walking siblings from room-type anchors. This handles hotels where the first rate +# row of a room type lacks js-rt-block-row, or where the room header uses a
+# instead of for the room_type_id_ element — both caused under-counting. _EXTRACT_RATES_JS = """ () => { const results = []; + const roomTypeNames = {}; - document.querySelectorAll('[id^="room_type_id_"]').forEach(roomEl => { - const roomId = roomEl.getAttribute('data-room-id') || roomEl.id.replace('room_type_id_', ''); - const roomName = ( - roomEl.querySelector('.hprt-roomtype-icon-link')?.innerText || - roomEl.querySelector('span')?.innerText || '' - ).trim(); + const rows = document.querySelectorAll( + '#available_rooms tbody tr:not([data-is-room-upgrade])' + ); - // Availability count from scarcity indicator ("We have 2 left") - const availText = roomEl.closest('tr') - ?.querySelector('.only_x_left, .thisRoomAvailabilityNew span') + for (const tr of rows) { + const blockId = tr.getAttribute('data-block-id') || ''; + if (!blockId) continue; + + const blockParts = blockId.split('_'); + if (blockParts.length < 3) continue; + + const roomId = blockParts[0]; + const maxPersons = parseInt(blockParts[2]); + + // Room name: the first row for each room type carries the room_type_id_ element. + // Subsequent rows for the same room type don't — reuse stored name (same as scrapy). + const roomNameEl = tr.querySelector('[id^="room_type_id_"]'); + if (roomNameEl) { + const name = ( + roomNameEl.querySelector('.hprt-roomtype-icon-link')?.innerText || + roomNameEl.querySelector('span')?.innerText || + roomNameEl.innerText || + '' + ).trim(); + if (name) roomTypeNames[roomId] = name; + } + const roomName = roomTypeNames[roomId] || ''; + + const priceRaw = tr.getAttribute('data-hotel-rounded-price') || ''; + + let fltrs = {}; + try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {} + + // Conditions cell (3rd ) holds meal plan + cancel info + const cells = tr.querySelectorAll('td'); + const condCell = cells.length >= 3 ? cells[2].innerText || '' : ''; + + const breakfastIncluded = condCell.toLowerCase().includes('breakfast'); + const nonRefundable = (fltrs.non_refundable === 1); + + const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i); + const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null; + + // Scarcity text ("We have 2 left") as fallback for availability + const availText = tr.querySelector('.only_x_left, .thisRoomAvailabilityNew span') ?.innerText?.trim() || ''; const availMatch = availText.match(/\\d+/); - const availCount = availMatch ? parseInt(availMatch[0]) : null; + let qtyAvailable = availMatch ? parseInt(availMatch[0]) : null; - // Walk sibling rows that belong to this room type - let tr = roomEl.closest('tr'); - while (tr) { - if (tr.classList.contains('js-rt-block-row')) { - const blockId = tr.getAttribute('data-block-id') || ''; - const priceRaw = tr.getAttribute('data-hotel-rounded-price') || ''; - - let fltrs = {}; - try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {} - - // Conditions cell (3rd ) holds meal plan + cancel info - const cells = tr.querySelectorAll('td'); - const condCell = cells.length >= 3 ? cells[2].innerText || '' : ''; - - const breakfastIncluded = condCell.toLowerCase().includes('breakfast'); - const nonRefundable = (fltrs.non_refundable === 1); - - // Free cancellation date: "Free cancellation before DD Month YYYY" - const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i); - const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null; - - // Persons: extract from block_id format {room_id}_{rate_plan_id}_{persons}_{meal}_0 - // More reliable than cell text parsing which can pick up the price instead. - const blockParts = blockId.split('_'); - const maxPersons = blockParts.length >= 3 ? parseInt(blockParts[2]) : null; - - // Quantity dropdown: last has a