hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Use text-search approach for rate conditions (breakfast/cancel/payment)

Browse source 
Original scrapy approach: extract <li> condition lines from each rate plan row
and text-search for known keywords. No match = null (unknown), not assumed false.

Adds breakfast_text, cancel_text, payment_text columns to booking_com_rates.
Booleans now nullable (null = not mentioned, true/false = explicit signal).

JS extraction searches all <li> items in the row (falls back to newline-split
innerText if none). Breakfast: 'breakfast' keyword. Cancel: 'free cancellation',
'non-refundable', 'total cost to cancel', 'fully chargeable'. Payment: 'no
prepayment', 'pay at the property', 'pay online'. data-fltrs used as fallback.

API snapshot endpoint now returns breakfast/cancel/payment text strings. Old
boolean-only rows degrade gracefully to derived labels.

Modal plan rows replace the fixed meal|cancel|price column layout with a single
stacked conditions cell: 0-3 lines depending on what the page actually shows.
Breakfast green when 'included', cancel green when 'Free cancellation…'.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-14 18:10:00 +00:00
parent 69ecf1012e
commit 38b0c36923
6 changed files with 132 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

84
backend/services/scraper_backends/playwright_hotel_page.py
View file

@ -109,26 +109,61 @@ _EXTRACT_RATES_JS = """
const cells = tr.querySelectorAll('td');
// Scan ALL cells for breakfast — the conditions column index varies
// across hotel page templates (some hotels swap occupancy + conditions).
// Also check data-fltrs which carries a mealplan flag on some properties.
const rowText = (tr.innerText || '').toLowerCase();
const breakfastIncluded = rowText.includes('breakfast')
|| fltrs.mealplan === 1
|| fltrs.breakfast_included === 1;
// Extract condition detail lines using the same text-search approach as the
// original scrapy spider: grab <li> items from the row (Booking.com renders
// meal plan, cancellation policy and payment method each as a separate <li>),
// then search each line for known keywords. No match → null, not assumed false.
const liTexts = Array.from(tr.querySelectorAll('li'))
.map(li => (li.innerText || '').trim())
.filter(Boolean);
const detailLines = liTexts.length
? liTexts
: (tr.innerText || '').split('\\n').map(l => l.trim()).filter(l => l.length > 2 && l.length < 200);
const nonRefundable = (fltrs.non_refundable === 1);
let breakfastText = null;
let cancelText = null;
let paymentText = null;
// Cancellation text: find whichever cell mentions it.
let condCell = '';
for (const cell of cells) {
const t = cell.innerText || '';
if (t.toLowerCase().includes('free cancellation')) { condCell = t; break; }
for (const line of detailLines) {
const ll = line.toLowerCase();
if (!breakfastText && ll.includes('breakfast')) breakfastText = line;
if (!cancelText && (ll.includes('free cancellation') || ll.includes('non-refundable')
|| ll.includes('total cost to cancel') || ll.includes('fully chargeable')
|| ll.includes('partially refundable'))) cancelText = line;
if (!paymentText && (ll.includes('no prepayment') || ll.includes('pay at the property')
|| ll.includes('pay the property') || ll.includes('pay online')
|| ll.includes('pay now'))) paymentText = line;
}
if (!condCell && cells.length >= 3) condCell = cells[2].innerText || '';
const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
// Cross-check with structured data-fltrs — use as fallback when text is absent
const fltrsBreakfast = fltrs.mealplan === 1 || fltrs.breakfast_included === 1;
const fltrsNonRefundable = fltrs.non_refundable === 1;
if (!breakfastText && fltrsBreakfast) breakfastText = 'Breakfast included';
if (!cancelText && fltrsNonRefundable) cancelText = 'Non-refundable';
if (!cancelText && fltrs.non_refundable === 0) cancelText = 'Free cancellation';
// Derive booleans — text is authoritative, fltrs as fallback
let breakfastIncluded = null;
if (breakfastText) {
breakfastIncluded = breakfastText.toLowerCase().includes('included') || fltrsBreakfast;
} else if (fltrsBreakfast) {
breakfastIncluded = true;
}
let freeCancellation = null;
if (cancelText) {
freeCancellation = cancelText.toLowerCase().includes('free cancellation');
} else if (fltrsNonRefundable) {
freeCancellation = false;
}
let noPrepayment = null;
if (paymentText) {
const pll = paymentText.toLowerCase();
noPrepayment = pll.includes('no prepayment') || pll.includes('pay at the property')
|| pll.includes('pay the property');
}
// Max persons: read from visible span text (same as original scrapy spider).
// block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy.
@ -175,9 +210,12 @@ _EXTRACT_RATES_JS = """
avail_count: qtyAvailable,
block_id: blockId,
price: priceRaw ? parseInt(priceRaw) : null,
breakfast_text: breakfastText,
cancel_text: cancelText,
payment_text: paymentText,
breakfast_included: breakfastIncluded,
non_refundable: nonRefundable,
free_cancel_text: freeCancelText,
free_cancellation: freeCancellation,
no_prepayment: noPrepayment,
max_persons: maxPersons,
});
}
@ -376,12 +414,16 @@ class PlaywrightHotelPageBackend(ScraperBackend):
rate_gross=Decimal(plan['price']),
currency='GBP',
room_type=plan['room_name'] or None,
breakfast_included=plan['breakfast_included'],
free_cancellation=not plan['non_refundable'],
rooms_left=plan['avail_count'], # qty from dropdown (max 10) or scarcity text
breakfast_included=plan['breakfast_included'], # True/False/None
free_cancellation=plan['free_cancellation'], # True/False/None
no_prepayment=plan['no_prepayment'], # True/False/None
rooms_left=plan['avail_count'],
available_qty=plan['avail_count'],
rate_plan_id=plan['block_id'] or None,
max_persons=plan['max_persons'],
breakfast_text=plan['breakfast_text'],
cancel_text=plan['cancel_text'],
payment_text=plan['payment_text'],
))
logger.info(