From 38b0c369234f34e2bc929f81b3bc8bd66216a977 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Tue, 14 Jul 2026 18:10:00 +0000 Subject: [PATCH] Use text-search approach for rate conditions (breakfast/cancel/payment) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original scrapy approach: extract
  • condition lines from each rate plan row and text-search for known keywords. No match = null (unknown), not assumed false. Adds breakfast_text, cancel_text, payment_text columns to booking_com_rates. Booleans now nullable (null = not mentioned, true/false = explicit signal). JS extraction searches all
  • items in the row (falls back to newline-split innerText if none). Breakfast: 'breakfast' keyword. Cancel: 'free cancellation', 'non-refundable', 'total cost to cancel', 'fully chargeable'. Payment: 'no prepayment', 'pay at the property', 'pay online'. data-fltrs used as fallback. API snapshot endpoint now returns breakfast/cancel/payment text strings. Old boolean-only rows degrade gracefully to derived labels. Modal plan rows replace the fixed meal|cancel|price column layout with a single stacked conditions cell: 0-3 lines depending on what the page actually shows. Breakfast green when 'included', cancel green when 'Free cancellation…'. Co-Authored-By: Claude Sonnet 4.6 --- backend/api/competitors.py | 48 +++++++---- backend/schema.sql | 10 +++ backend/services/booking_scraper.py | 9 +- backend/services/scraper_backends/base.py | 3 + .../scraper_backends/playwright_hotel_page.py | 84 ++++++++++++++----- frontend/src/pages/MarketView.tsx | 24 ++++-- 6 files changed, 132 insertions(+), 46 deletions(-) diff --git a/backend/api/competitors.py b/backend/api/competitors.py index de5bdac..3379f33 100644 --- a/backend/api/competitors.py +++ b/backend/api/competitors.py @@ -1368,10 +1368,14 @@ async def get_hotel_rate_snapshot( r.room_type, r.breakfast_included, r.free_cancellation, + r.no_prepayment, r.rate_gross::float AS price, r.rooms_left, r.max_persons, - r.availability_status + r.availability_status, + r.breakfast_text, + r.cancel_text, + r.payment_text FROM booking_com_rates r WHERE r.hotel_id = :hotel_id AND r.rate_date = :stay_date @@ -1383,22 +1387,41 @@ async def get_hotel_rate_snapshot( ORDER BY scraped_at DESC LIMIT 1 ) - ORDER BY r.room_type, r.breakfast_included, r.free_cancellation DESC, r.rate_gross + ORDER BY r.room_type, r.breakfast_included NULLS LAST, r.free_cancellation DESC NULLS LAST, r.rate_gross """), {"hotel_id": hotel_id, "stay_date": stay_date}, ) rows = result.mappings().all() - # If no rate_plan data (old scraper rows), return a simple summary if not rows: return {"stay_date": str(stay_date), "rooms": [], "legacy": True} - # Check if this is old single-row data (no rate_plan_id breakdown) has_breakdown = any(r["room_type"] for r in rows) + def _plan(r) -> dict: + # Use stored text when available; fall back to deriving from booleans for old rows. + breakfast = r["breakfast_text"] or ( + "Breakfast included" if r["breakfast_included"] is True else + "Breakfast available as extra" if r["breakfast_included"] is False else None + ) + cancel = r["cancel_text"] or ( + "Free cancellation" if r["free_cancellation"] is True else + "Non-refundable" if r["free_cancellation"] is False else None + ) + payment = r["payment_text"] or ( + "No prepayment needed – pay at the property" if r["no_prepayment"] is True else + "Pay online" if r["no_prepayment"] is False else None + ) + return { + "breakfast": breakfast, + "cancel": cancel, + "payment": payment, + "price": r["price"], + "max_persons": r["max_persons"], + } + if not has_breakdown: - # Legacy: single row per hotel+date from search results scraper row = rows[0] return { "stay_date": str(stay_date), @@ -1407,16 +1430,10 @@ async def get_hotel_rate_snapshot( "room_type": "Best available", "rooms_left": row["rooms_left"], "availability_status": row["availability_status"], - "plans": [{ - "meal": "B&B" if row["breakfast_included"] else "Room only", - "cancel": "Free cancellation" if row["free_cancellation"] else "Non-refundable", - "price": row["price"], - "max_persons": row["max_persons"], - }] if row["price"] else [], + "plans": [_plan(row)] if row["price"] else [], }], } - # Group by room_type, keeping rooms_left from the first occurrence (same per room) rooms_map: dict = {} for r in rows: rt = (r["room_type"] or "Unknown").strip() @@ -1428,12 +1445,7 @@ async def get_hotel_rate_snapshot( "plans": [], } if r["price"] and (r["max_persons"] is None or r["max_persons"] == 2): - rooms_map[rt]["plans"].append({ - "meal": "B&B" if r["breakfast_included"] else "Room only", - "cancel": "Free cancellation" if r["free_cancellation"] else "Non-refundable", - "price": r["price"], - "max_persons": r["max_persons"], - }) + rooms_map[rt]["plans"].append(_plan(r)) return { "stay_date": str(stay_date), diff --git a/backend/schema.sql b/backend/schema.sql index 45eb8dc..3c28a53 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -133,6 +133,16 @@ ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255); ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS rate_plan_id TEXT; ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS max_persons INTEGER; +-- Raw condition text from the page (text search approach — null = not mentioned, not "no") +ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS breakfast_text TEXT; +ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS cancel_text TEXT; +ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS payment_text TEXT; + +-- Make condition booleans nullable (null = unknown, not assumed false) +ALTER TABLE booking_com_rates ALTER COLUMN breakfast_included SET DEFAULT NULL; +ALTER TABLE booking_com_rates ALTER COLUMN free_cancellation SET DEFAULT NULL; +ALTER TABLE booking_com_rates ALTER COLUMN no_prepayment SET DEFAULT NULL; + CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC); diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index 32e8cc8..812493a 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -198,10 +198,12 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID): INSERT INTO booking_com_rates (hotel_id, rate_date, availability_status, rate_gross, currency, room_type, breakfast_included, free_cancellation, no_prepayment, rooms_left, - rate_plan_id, max_persons, scrape_batch_id) + rate_plan_id, max_persons, scrape_batch_id, + breakfast_text, cancel_text, payment_text) VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type, :breakfast, :cancel, :prepay, :rooms_left, - :rate_plan_id, :max_persons, :batch_id) + :rate_plan_id, :max_persons, :batch_id, + :breakfast_text, :cancel_text, :payment_text) """), { 'hotel_id': hotel_id, @@ -217,6 +219,9 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID): 'rate_plan_id': rate.rate_plan_id, 'max_persons': rate.max_persons, 'batch_id': str(batch_id), + 'breakfast_text': rate.breakfast_text, + 'cancel_text': rate.cancel_text, + 'payment_text': rate.payment_text, } ) diff --git a/backend/services/scraper_backends/base.py b/backend/services/scraper_backends/base.py index 34f0b43..d286f73 100644 --- a/backend/services/scraper_backends/base.py +++ b/backend/services/scraper_backends/base.py @@ -38,6 +38,9 @@ class RateData: available_qty: Optional[int] = None # Future: from hotel page dropdown rate_plan_id: Optional[str] = None # block_id from hotel page (identifies rate variant) max_persons: Optional[int] = None # Occupancy this rate applies to + breakfast_text: Optional[str] = None # e.g. "Superb breakfast included" / "Superb breakfast £17.50" + cancel_text: Optional[str] = None # e.g. "Free cancellation before 15 July 2026" / "Non-refundable" + payment_text: Optional[str] = None # e.g. "No prepayment needed – pay at the property" / "Pay online" @dataclass diff --git a/backend/services/scraper_backends/playwright_hotel_page.py b/backend/services/scraper_backends/playwright_hotel_page.py index 64cd04b..43f3b3c 100644 --- a/backend/services/scraper_backends/playwright_hotel_page.py +++ b/backend/services/scraper_backends/playwright_hotel_page.py @@ -109,26 +109,61 @@ _EXTRACT_RATES_JS = """ const cells = tr.querySelectorAll('td'); - // Scan ALL cells for breakfast — the conditions column index varies - // across hotel page templates (some hotels swap occupancy + conditions). - // Also check data-fltrs which carries a mealplan flag on some properties. - const rowText = (tr.innerText || '').toLowerCase(); - const breakfastIncluded = rowText.includes('breakfast') - || fltrs.mealplan === 1 - || fltrs.breakfast_included === 1; + // Extract condition detail lines using the same text-search approach as the + // original scrapy spider: grab
  • items from the row (Booking.com renders + // meal plan, cancellation policy and payment method each as a separate
  • ), + // then search each line for known keywords. No match → null, not assumed false. + const liTexts = Array.from(tr.querySelectorAll('li')) + .map(li => (li.innerText || '').trim()) + .filter(Boolean); + const detailLines = liTexts.length + ? liTexts + : (tr.innerText || '').split('\\n').map(l => l.trim()).filter(l => l.length > 2 && l.length < 200); - const nonRefundable = (fltrs.non_refundable === 1); + let breakfastText = null; + let cancelText = null; + let paymentText = null; - // Cancellation text: find whichever cell mentions it. - let condCell = ''; - for (const cell of cells) { - const t = cell.innerText || ''; - if (t.toLowerCase().includes('free cancellation')) { condCell = t; break; } + for (const line of detailLines) { + const ll = line.toLowerCase(); + if (!breakfastText && ll.includes('breakfast')) breakfastText = line; + if (!cancelText && (ll.includes('free cancellation') || ll.includes('non-refundable') + || ll.includes('total cost to cancel') || ll.includes('fully chargeable') + || ll.includes('partially refundable'))) cancelText = line; + if (!paymentText && (ll.includes('no prepayment') || ll.includes('pay at the property') + || ll.includes('pay the property') || ll.includes('pay online') + || ll.includes('pay now'))) paymentText = line; } - if (!condCell && cells.length >= 3) condCell = cells[2].innerText || ''; - const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i); - const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null; + // Cross-check with structured data-fltrs — use as fallback when text is absent + const fltrsBreakfast = fltrs.mealplan === 1 || fltrs.breakfast_included === 1; + const fltrsNonRefundable = fltrs.non_refundable === 1; + + if (!breakfastText && fltrsBreakfast) breakfastText = 'Breakfast included'; + if (!cancelText && fltrsNonRefundable) cancelText = 'Non-refundable'; + if (!cancelText && fltrs.non_refundable === 0) cancelText = 'Free cancellation'; + + // Derive booleans — text is authoritative, fltrs as fallback + let breakfastIncluded = null; + if (breakfastText) { + breakfastIncluded = breakfastText.toLowerCase().includes('included') || fltrsBreakfast; + } else if (fltrsBreakfast) { + breakfastIncluded = true; + } + + let freeCancellation = null; + if (cancelText) { + freeCancellation = cancelText.toLowerCase().includes('free cancellation'); + } else if (fltrsNonRefundable) { + freeCancellation = false; + } + + let noPrepayment = null; + if (paymentText) { + const pll = paymentText.toLowerCase(); + noPrepayment = pll.includes('no prepayment') || pll.includes('pay at the property') + || pll.includes('pay the property'); + } // Max persons: read from visible span text (same as original scrapy spider). // block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy. @@ -175,9 +210,12 @@ _EXTRACT_RATES_JS = """ avail_count: qtyAvailable, block_id: blockId, price: priceRaw ? parseInt(priceRaw) : null, + breakfast_text: breakfastText, + cancel_text: cancelText, + payment_text: paymentText, breakfast_included: breakfastIncluded, - non_refundable: nonRefundable, - free_cancel_text: freeCancelText, + free_cancellation: freeCancellation, + no_prepayment: noPrepayment, max_persons: maxPersons, }); } @@ -376,12 +414,16 @@ class PlaywrightHotelPageBackend(ScraperBackend): rate_gross=Decimal(plan['price']), currency='GBP', room_type=plan['room_name'] or None, - breakfast_included=plan['breakfast_included'], - free_cancellation=not plan['non_refundable'], - rooms_left=plan['avail_count'], # qty from dropdown (max 10) or scarcity text + breakfast_included=plan['breakfast_included'], # True/False/None + free_cancellation=plan['free_cancellation'], # True/False/None + no_prepayment=plan['no_prepayment'], # True/False/None + rooms_left=plan['avail_count'], available_qty=plan['avail_count'], rate_plan_id=plan['block_id'] or None, max_persons=plan['max_persons'], + breakfast_text=plan['breakfast_text'], + cancel_text=plan['cancel_text'], + payment_text=plan['payment_text'], )) logger.info( diff --git a/frontend/src/pages/MarketView.tsx b/frontend/src/pages/MarketView.tsx index 56d17ee..b384e4a 100644 --- a/frontend/src/pages/MarketView.tsx +++ b/frontend/src/pages/MarketView.tsx @@ -1276,7 +1276,7 @@ const MonthSelector: React.FC<{ interface RateHistoryPoint { t: string | null; rate: number } interface RateHistorySeries { room_type: string; points: RateHistoryPoint[] } interface RateHistoryModal { hotelId: number; hotelName: string; stayDate: string } -interface RatePlan { meal: string; cancel: string; price: number; max_persons: number | null } +interface RatePlan { breakfast: string | null; cancel: string | null; payment: string | null; price: number; max_persons: number | null } interface RoomSnapshot { room_type: string; rooms_left: number | null; availability_status: string; plans: RatePlan[] } interface RateSnapshot { stay_date: string; legacy: boolean; rooms: RoomSnapshot[] } @@ -1350,11 +1350,25 @@ const RateHistoryModalComponent: React.FC<{ modal: RateHistoryModal; onClose: () {room.plans.map((plan, pi) => ( - {plan.meal} - - {plan.cancel} + + {plan.breakfast && ( +
    + {plan.breakfast} +
    + )} + {plan.cancel && ( +
    + {plan.cancel} +
    + )} + {plan.payment && ( +
    {plan.payment}
    + )} + {!plan.breakfast && !plan.cancel && !plan.payment && ( + — + )} - + £{plan.price.toLocaleString()}