hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Fix JS extractor under-counting rooms: iterate all tbody rows like scrapy

Browse source 
The previous approach walked siblings from [id^="room_type_id_"] anchors
and only processed rows with js-rt-block-row class, causing the first rate
row of some room types to be silently skipped when that class was absent.

Now iterates all #available_rooms tbody tr rows with data-block-id (same
strategy as the original scrapy spider), using the room_type_id_ element
presence to identify room names only on the first row, with fallback to the
stored name for subsequent rows of the same room type.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-10 00:37:35 +00:00
parent dd84ca230a
commit 83b46bee8f
1 changed files with 78 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

154
backend/services/scraper_backends/playwright_hotel_page.py
View file

@ -36,93 +36,95 @@ logger = logging.getLogger(__name__)
# JS that extracts all rate plan rows from the room availability table. # JS that extracts all rate plan rows from the room availability table.
# Runs inside the page after the room table has loaded. # Runs inside the page after the room table has loaded.
#
# Iterates ALL tbody rows (matching the original scrapy spider approach) rather than
# walking siblings from room-type anchors. This handles hotels where the first rate
# row of a room type lacks js-rt-block-row, or where the room header uses a <div>
# instead of <a> for the room_type_id_ element — both caused under-counting.
_EXTRACT_RATES_JS = """ _EXTRACT_RATES_JS = """
() => { () => {
const results = []; const results = [];
const roomTypeNames = {};
document.querySelectorAll('[id^="room_type_id_"]').forEach(roomEl => { const rows = document.querySelectorAll(
const roomId = roomEl.getAttribute('data-room-id') || roomEl.id.replace('room_type_id_', ''); '#available_rooms tbody tr:not([data-is-room-upgrade])'
const roomName = ( );
roomEl.querySelector('.hprt-roomtype-icon-link')?.innerText ||
roomEl.querySelector('span')?.innerText || ''
).trim();
// Availability count from scarcity indicator ("We have 2 left") for (const tr of rows) {
const availText = roomEl.closest('tr') const blockId = tr.getAttribute('data-block-id') || '';
?.querySelector('.only_x_left, .thisRoomAvailabilityNew span') if (!blockId) continue;
const blockParts = blockId.split('_');
if (blockParts.length < 3) continue;
const roomId = blockParts[0];
const maxPersons = parseInt(blockParts[2]);
// Room name: the first row for each room type carries the room_type_id_ element.
// Subsequent rows for the same room type don't — reuse stored name (same as scrapy).
const roomNameEl = tr.querySelector('[id^="room_type_id_"]');
if (roomNameEl) {
const name = (
roomNameEl.querySelector('.hprt-roomtype-icon-link')?.innerText ||
roomNameEl.querySelector('span')?.innerText ||
roomNameEl.innerText ||
''
).trim();
if (name) roomTypeNames[roomId] = name;
}
const roomName = roomTypeNames[roomId] || '';
const priceRaw = tr.getAttribute('data-hotel-rounded-price') || '';
let fltrs = {};
try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {}
// Conditions cell (3rd <td>) holds meal plan + cancel info
const cells = tr.querySelectorAll('td');
const condCell = cells.length >= 3 ? cells[2].innerText || '' : '';
const breakfastIncluded = condCell.toLowerCase().includes('breakfast');
const nonRefundable = (fltrs.non_refundable === 1);
const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
// Scarcity text ("We have 2 left") as fallback for availability
const availText = tr.querySelector('.only_x_left, .thisRoomAvailabilityNew span')
?.innerText?.trim() || ''; ?.innerText?.trim() || '';
const availMatch = availText.match(/\\d+/); const availMatch = availText.match(/\\d+/);
const availCount = availMatch ? parseInt(availMatch[0]) : null; let qtyAvailable = availMatch ? parseInt(availMatch[0]) : null;
// Walk sibling <tr> rows that belong to this room type // Quantity dropdown in last <td>: options 0..N where N = available qty (capped at 10)
let tr = roomEl.closest('tr'); const qtyCell = cells.length >= 2 ? cells[cells.length - 1] : null;
while (tr) { if (qtyCell) {
if (tr.classList.contains('js-rt-block-row')) { const sel = qtyCell.querySelector('select');
const blockId = tr.getAttribute('data-block-id') || ''; if (sel) {
const priceRaw = tr.getAttribute('data-hotel-rounded-price') || ''; const vals = Array.from(sel.options)
.map(o => parseInt(o.value))
let fltrs = {}; .filter(v => !isNaN(v) && v > 0);
try { fltrs = JSON.parse(tr.getAttribute('data-fltrs') || '{}'); } catch(e) {} if (vals.length > 0) qtyAvailable = Math.max(...vals);
} else {
// Conditions cell (3rd <td>) holds meal plan + cancel info const nums = (qtyCell.innerText || '').match(/^(\\d+)/gm);
const cells = tr.querySelectorAll('td'); if (nums && nums.length > 0) {
const condCell = cells.length >= 3 ? cells[2].innerText || '' : ''; const parsed = nums.map(n => parseInt(n)).filter(v => v > 0);
if (parsed.length > 0) qtyAvailable = Math.max(...parsed);
const breakfastIncluded = condCell.toLowerCase().includes('breakfast');
const nonRefundable = (fltrs.non_refundable === 1);
// Free cancellation date: "Free cancellation before DD Month YYYY"
const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
// Persons: extract from block_id format {room_id}_{rate_plan_id}_{persons}_{meal}_0
// More reliable than cell text parsing which can pick up the price instead.
const blockParts = blockId.split('_');
const maxPersons = blockParts.length >= 3 ? parseInt(blockParts[2]) : null;
// Quantity dropdown: last <td> has a <select> with options 0..N
// where N = actual available qty (capped at 10 by Booking.com).
// This is more reliable than the "X left" scarcity text which only
// appears when availability is low (typically ≤5).
let qtyAvailable = availCount; // fallback to scarcity text
const qtyCell = cells.length >= 4 ? cells[cells.length - 1] : null;
if (qtyCell) {
const sel = qtyCell.querySelector('select');
if (sel) {
const vals = Array.from(sel.options)
.map(o => parseInt(o.value))
.filter(v => !isNaN(v) && v > 0);
if (vals.length > 0) qtyAvailable = Math.max(...vals);
} else {
// Fallback: parse max number from cell text
// "Select rooms\\n0\\n1 (£208)\\n2 (£416)" → [1,2] → max 2
const nums = (qtyCell.innerText || '').match(/^(\\d+)/gm);
if (nums && nums.length > 0) {
const parsed = nums.map(n => parseInt(n)).filter(v => v > 0);
if (parsed.length > 0) qtyAvailable = Math.max(...parsed);
}
}
} }
results.push({
room_id: roomId,
room_name: roomName,
avail_count: qtyAvailable,
block_id: blockId,
price: priceRaw ? parseInt(priceRaw) : null,
breakfast_included: breakfastIncluded,
non_refundable: nonRefundable,
free_cancel_text: freeCancelText,
max_persons: maxPersons,
});
} }
tr = tr.nextElementSibling;
if (!tr) break;
// Stop at the next room type's header row
if (tr.querySelector('[id^="room_type_id_"]')) break;
} }
});
results.push({
room_id: roomId,
room_name: roomName,
avail_count: qtyAvailable,
block_id: blockId,
price: priceRaw ? parseInt(priceRaw) : null,
breakfast_included: breakfastIncluded,
non_refundable: nonRefundable,
free_cancel_text: freeCancelText,
max_persons: !isNaN(maxPersons) ? maxPersons : null,
});
}
return results; return results;
} }