hotel-manage-stack/rates
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Use text-search approach for rate conditions (breakfast/cancel/payment)

Browse source 
Original scrapy approach: extract <li> condition lines from each rate plan row
and text-search for known keywords. No match = null (unknown), not assumed false.

Adds breakfast_text, cancel_text, payment_text columns to booking_com_rates.
Booleans now nullable (null = not mentioned, true/false = explicit signal).

JS extraction searches all <li> items in the row (falls back to newline-split
innerText if none). Breakfast: 'breakfast' keyword. Cancel: 'free cancellation',
'non-refundable', 'total cost to cancel', 'fully chargeable'. Payment: 'no
prepayment', 'pay at the property', 'pay online'. data-fltrs used as fallback.

API snapshot endpoint now returns breakfast/cancel/payment text strings. Old
boolean-only rows degrade gracefully to derived labels.

Modal plan rows replace the fixed meal|cancel|price column layout with a single
stacked conditions cell: 0-3 lines depending on what the page actually shows.
Breakfast green when 'included', cancel green when 'Free cancellation…'.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-14 18:10:00 +00:00
parent 69ecf1012e
commit 38b0c36923
6 changed files with 132 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

48
backend/api/competitors.py
View file

@ -1368,10 +1368,14 @@ async def get_hotel_rate_snapshot(
r.room_type,
r.breakfast_included,
r.free_cancellation,
r.no_prepayment,
r.rate_gross::float AS price,
r.rooms_left,
r.max_persons,
r.availability_status
r.availability_status,
r.breakfast_text,
r.cancel_text,
r.payment_text
FROM booking_com_rates r
WHERE r.hotel_id = :hotel_id
AND r.rate_date = :stay_date
@ -1383,22 +1387,41 @@ async def get_hotel_rate_snapshot(
ORDER BY scraped_at DESC
LIMIT 1
)
ORDER BY r.room_type, r.breakfast_included, r.free_cancellation DESC, r.rate_gross
ORDER BY r.room_type, r.breakfast_included NULLS LAST, r.free_cancellation DESC NULLS LAST, r.rate_gross
"""),
{"hotel_id": hotel_id, "stay_date": stay_date},
)
rows = result.mappings().all()
# If no rate_plan data (old scraper rows), return a simple summary
if not rows:
return {"stay_date": str(stay_date), "rooms": [], "legacy": True}
# Check if this is old single-row data (no rate_plan_id breakdown)
has_breakdown = any(r["room_type"] for r in rows)
def _plan(r) -> dict:
# Use stored text when available; fall back to deriving from booleans for old rows.
breakfast = r["breakfast_text"] or (
"Breakfast included" if r["breakfast_included"] is True else
"Breakfast available as extra" if r["breakfast_included"] is False else None
)
cancel = r["cancel_text"] or (
"Free cancellation" if r["free_cancellation"] is True else
"Non-refundable" if r["free_cancellation"] is False else None
)
payment = r["payment_text"] or (
"No prepayment needed – pay at the property" if r["no_prepayment"] is True else
"Pay online" if r["no_prepayment"] is False else None
)
return {
"breakfast": breakfast,
"cancel": cancel,
"payment": payment,
"price": r["price"],
"max_persons": r["max_persons"],
}
if not has_breakdown:
# Legacy: single row per hotel+date from search results scraper
row = rows[0]
return {
"stay_date": str(stay_date),
@ -1407,16 +1430,10 @@ async def get_hotel_rate_snapshot(
"room_type": "Best available",
"rooms_left": row["rooms_left"],
"availability_status": row["availability_status"],
"plans": [{
"meal": "B&B" if row["breakfast_included"] else "Room only",
"cancel": "Free cancellation" if row["free_cancellation"] else "Non-refundable",
"price": row["price"],
"max_persons": row["max_persons"],
}] if row["price"] else [],
"plans": [_plan(row)] if row["price"] else [],
}],
}
# Group by room_type, keeping rooms_left from the first occurrence (same per room)
rooms_map: dict = {}
for r in rows:
rt = (r["room_type"] or "Unknown").strip()
@ -1428,12 +1445,7 @@ async def get_hotel_rate_snapshot(
"plans": [],
}
if r["price"] and (r["max_persons"] is None or r["max_persons"] == 2):
rooms_map[rt]["plans"].append({
"meal": "B&B" if r["breakfast_included"] else "Room only",
"cancel": "Free cancellation" if r["free_cancellation"] else "Non-refundable",
"price": r["price"],
"max_persons": r["max_persons"],
})
rooms_map[rt]["plans"].append(_plan(r))
return {
"stay_date": str(stay_date),

10
backend/schema.sql
View file

@ -133,6 +133,16 @@ ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255);
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS rate_plan_id TEXT;
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS max_persons INTEGER;
-- Raw condition text from the page (text search approach — null = not mentioned, not "no")
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS breakfast_text TEXT;
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS cancel_text TEXT;
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS payment_text TEXT;
-- Make condition booleans nullable (null = unknown, not assumed false)
ALTER TABLE booking_com_rates ALTER COLUMN breakfast_included SET DEFAULT NULL;
ALTER TABLE booking_com_rates ALTER COLUMN free_cancellation SET DEFAULT NULL;
ALTER TABLE booking_com_rates ALTER COLUMN no_prepayment SET DEFAULT NULL;
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);

9
backend/services/booking_scraper.py
View file

@ -198,10 +198,12 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
INSERT INTO booking_com_rates
(hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
breakfast_included, free_cancellation, no_prepayment, rooms_left,
rate_plan_id, max_persons, scrape_batch_id)
rate_plan_id, max_persons, scrape_batch_id,
breakfast_text, cancel_text, payment_text)
VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
:breakfast, :cancel, :prepay, :rooms_left,
:rate_plan_id, :max_persons, :batch_id)
:rate_plan_id, :max_persons, :batch_id,
:breakfast_text, :cancel_text, :payment_text)
"""),
{
'hotel_id': hotel_id,
@ -217,6 +219,9 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
'rate_plan_id': rate.rate_plan_id,
'max_persons': rate.max_persons,
'batch_id': str(batch_id),
'breakfast_text': rate.breakfast_text,
'cancel_text': rate.cancel_text,
'payment_text': rate.payment_text,
}
)

3
backend/services/scraper_backends/base.py
View file

@ -38,6 +38,9 @@ class RateData:
available_qty: Optional[int] = None # Future: from hotel page dropdown
rate_plan_id: Optional[str] = None # block_id from hotel page (identifies rate variant)
max_persons: Optional[int] = None # Occupancy this rate applies to
breakfast_text: Optional[str] = None # e.g. "Superb breakfast included" / "Superb breakfast £17.50"
cancel_text: Optional[str] = None # e.g. "Free cancellation before 15 July 2026" / "Non-refundable"
payment_text: Optional[str] = None # e.g. "No prepayment needed – pay at the property" / "Pay online"
@dataclass

84
backend/services/scraper_backends/playwright_hotel_page.py
View file

@ -109,26 +109,61 @@ _EXTRACT_RATES_JS = """
const cells = tr.querySelectorAll('td');
// Scan ALL cells for breakfast — the conditions column index varies
// across hotel page templates (some hotels swap occupancy + conditions).
// Also check data-fltrs which carries a mealplan flag on some properties.
const rowText = (tr.innerText || '').toLowerCase();
const breakfastIncluded = rowText.includes('breakfast')
|| fltrs.mealplan === 1
|| fltrs.breakfast_included === 1;
// Extract condition detail lines using the same text-search approach as the
// original scrapy spider: grab <li> items from the row (Booking.com renders
// meal plan, cancellation policy and payment method each as a separate <li>),
// then search each line for known keywords. No match → null, not assumed false.
const liTexts = Array.from(tr.querySelectorAll('li'))
.map(li => (li.innerText || '').trim())
.filter(Boolean);
const detailLines = liTexts.length
? liTexts
: (tr.innerText || '').split('\\n').map(l => l.trim()).filter(l => l.length > 2 && l.length < 200);
const nonRefundable = (fltrs.non_refundable === 1);
let breakfastText = null;
let cancelText = null;
let paymentText = null;
// Cancellation text: find whichever cell mentions it.
let condCell = '';
for (const cell of cells) {
const t = cell.innerText || '';
if (t.toLowerCase().includes('free cancellation')) { condCell = t; break; }
for (const line of detailLines) {
const ll = line.toLowerCase();
if (!breakfastText && ll.includes('breakfast')) breakfastText = line;
if (!cancelText && (ll.includes('free cancellation') || ll.includes('non-refundable')
|| ll.includes('total cost to cancel') || ll.includes('fully chargeable')
|| ll.includes('partially refundable'))) cancelText = line;
if (!paymentText && (ll.includes('no prepayment') || ll.includes('pay at the property')
|| ll.includes('pay the property') || ll.includes('pay online')
|| ll.includes('pay now'))) paymentText = line;
}
if (!condCell && cells.length >= 3) condCell = cells[2].innerText || '';
const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
// Cross-check with structured data-fltrs — use as fallback when text is absent
const fltrsBreakfast = fltrs.mealplan === 1 || fltrs.breakfast_included === 1;
const fltrsNonRefundable = fltrs.non_refundable === 1;
if (!breakfastText && fltrsBreakfast) breakfastText = 'Breakfast included';
if (!cancelText && fltrsNonRefundable) cancelText = 'Non-refundable';
if (!cancelText && fltrs.non_refundable === 0) cancelText = 'Free cancellation';
// Derive booleans — text is authoritative, fltrs as fallback
let breakfastIncluded = null;
if (breakfastText) {
breakfastIncluded = breakfastText.toLowerCase().includes('included') || fltrsBreakfast;
} else if (fltrsBreakfast) {
breakfastIncluded = true;
}
let freeCancellation = null;
if (cancelText) {
freeCancellation = cancelText.toLowerCase().includes('free cancellation');
} else if (fltrsNonRefundable) {
freeCancellation = false;
}
let noPrepayment = null;
if (paymentText) {
const pll = paymentText.toLowerCase();
noPrepayment = pll.includes('no prepayment') || pll.includes('pay at the property')
|| pll.includes('pay the property');
}
// Max persons: read from visible span text (same as original scrapy spider).
// block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy.
@ -175,9 +210,12 @@ _EXTRACT_RATES_JS = """
avail_count: qtyAvailable,
block_id: blockId,
price: priceRaw ? parseInt(priceRaw) : null,
breakfast_text: breakfastText,
cancel_text: cancelText,
payment_text: paymentText,
breakfast_included: breakfastIncluded,
non_refundable: nonRefundable,
free_cancel_text: freeCancelText,
free_cancellation: freeCancellation,
no_prepayment: noPrepayment,
max_persons: maxPersons,
});
}
@ -376,12 +414,16 @@ class PlaywrightHotelPageBackend(ScraperBackend):
rate_gross=Decimal(plan['price']),
currency='GBP',
room_type=plan['room_name'] or None,
breakfast_included=plan['breakfast_included'],
free_cancellation=not plan['non_refundable'],
rooms_left=plan['avail_count'], # qty from dropdown (max 10) or scarcity text
breakfast_included=plan['breakfast_included'], # True/False/None
free_cancellation=plan['free_cancellation'], # True/False/None
no_prepayment=plan['no_prepayment'], # True/False/None
rooms_left=plan['avail_count'],
available_qty=plan['avail_count'],
rate_plan_id=plan['block_id'] or None,
max_persons=plan['max_persons'],
breakfast_text=plan['breakfast_text'],
cancel_text=plan['cancel_text'],
payment_text=plan['payment_text'],
))
logger.info(