Use text-search approach for rate conditions (breakfast/cancel/payment)

Original scrapy approach: extract <li> condition lines from each rate plan row
and text-search for known keywords. No match = null (unknown), not assumed false.

Adds breakfast_text, cancel_text, payment_text columns to booking_com_rates.
Booleans now nullable (null = not mentioned, true/false = explicit signal).

JS extraction searches all <li> items in the row (falls back to newline-split
innerText if none). Breakfast: 'breakfast' keyword. Cancel: 'free cancellation',
'non-refundable', 'total cost to cancel', 'fully chargeable'. Payment: 'no
prepayment', 'pay at the property', 'pay online'. data-fltrs used as fallback.

API snapshot endpoint now returns breakfast/cancel/payment text strings. Old
boolean-only rows degrade gracefully to derived labels.

Modal plan rows replace the fixed meal|cancel|price column layout with a single
stacked conditions cell: 0-3 lines depending on what the page actually shows.
Breakfast green when 'included', cancel green when 'Free cancellation…'.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/api/competitors.py

@@ -1368,10 +1368,14 @@ async def get_hotel_rate_snapshot(
                 r.room_type,
                 r.breakfast_included,
                 r.free_cancellation,
+                r.no_prepayment,
                 r.rate_gross::float AS price,
                 r.rooms_left,
                 r.max_persons,
-                r.availability_status
+                r.availability_status,
+                r.breakfast_text,
+                r.cancel_text,
+                r.payment_text
             FROM booking_com_rates r
             WHERE r.hotel_id = :hotel_id
               AND r.rate_date = :stay_date
@@ -1383,22 +1387,41 @@ async def get_hotel_rate_snapshot(
                   ORDER BY scraped_at DESC
                   LIMIT 1
               )
-            ORDER BY r.room_type, r.breakfast_included, r.free_cancellation DESC, r.rate_gross
+            ORDER BY r.room_type, r.breakfast_included NULLS LAST, r.free_cancellation DESC NULLS LAST, r.rate_gross
         """),
         {"hotel_id": hotel_id, "stay_date": stay_date},
     )
 
     rows = result.mappings().all()
 
-    # If no rate_plan data (old scraper rows), return a simple summary
     if not rows:
         return {"stay_date": str(stay_date), "rooms": [], "legacy": True}
 
-    # Check if this is old single-row data (no rate_plan_id breakdown)
     has_breakdown = any(r["room_type"] for r in rows)
 
+    def _plan(r) -> dict:
+        # Use stored text when available; fall back to deriving from booleans for old rows.
+        breakfast = r["breakfast_text"] or (
+            "Breakfast included" if r["breakfast_included"] is True else
+            "Breakfast available as extra" if r["breakfast_included"] is False else None
+        )
+        cancel = r["cancel_text"] or (
+            "Free cancellation" if r["free_cancellation"] is True else
+            "Non-refundable" if r["free_cancellation"] is False else None
+        )
+        payment = r["payment_text"] or (
+            "No prepayment needed – pay at the property" if r["no_prepayment"] is True else
+            "Pay online" if r["no_prepayment"] is False else None
+        )
+        return {
+            "breakfast": breakfast,
+            "cancel": cancel,
+            "payment": payment,
+            "price": r["price"],
+            "max_persons": r["max_persons"],
+        }
+
     if not has_breakdown:
-        # Legacy: single row per hotel+date from search results scraper
         row = rows[0]
         return {
             "stay_date": str(stay_date),
@@ -1407,16 +1430,10 @@ async def get_hotel_rate_snapshot(
                 "room_type": "Best available",
                 "rooms_left": row["rooms_left"],
                 "availability_status": row["availability_status"],
-                "plans": [{
+                "plans": [_plan(row)] if row["price"] else [],
-                    "meal": "B&B" if row["breakfast_included"] else "Room only",
-                    "cancel": "Free cancellation" if row["free_cancellation"] else "Non-refundable",
-                    "price": row["price"],
-                    "max_persons": row["max_persons"],
-                }] if row["price"] else [],
             }],
         }
 
-    # Group by room_type, keeping rooms_left from the first occurrence (same per room)
     rooms_map: dict = {}
     for r in rows:
         rt = (r["room_type"] or "Unknown").strip()
@@ -1428,12 +1445,7 @@ async def get_hotel_rate_snapshot(
                 "plans": [],
             }
         if r["price"] and (r["max_persons"] is None or r["max_persons"] == 2):
-            rooms_map[rt]["plans"].append({
+            rooms_map[rt]["plans"].append(_plan(r))
-                "meal": "B&B" if r["breakfast_included"] else "Room only",
-                "cancel": "Free cancellation" if r["free_cancellation"] else "Non-refundable",
-                "price": r["price"],
-                "max_persons": r["max_persons"],
-            })
 
     return {
         "stay_date": str(stay_date),

backend/schema.sql

@@ -133,6 +133,16 @@ ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255);
 ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS rate_plan_id TEXT;
 ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS max_persons  INTEGER;
 
+-- Raw condition text from the page (text search approach — null = not mentioned, not "no")
+ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS breakfast_text TEXT;
+ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS cancel_text    TEXT;
+ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS payment_text   TEXT;
+
+-- Make condition booleans nullable (null = unknown, not assumed false)
+ALTER TABLE booking_com_rates ALTER COLUMN breakfast_included SET DEFAULT NULL;
+ALTER TABLE booking_com_rates ALTER COLUMN free_cancellation  SET DEFAULT NULL;
+ALTER TABLE booking_com_rates ALTER COLUMN no_prepayment      SET DEFAULT NULL;
+
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
 CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);

backend/services/booking_scraper.py

@@ -198,10 +198,12 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
             INSERT INTO booking_com_rates
             (hotel_id, rate_date, availability_status, rate_gross, currency, room_type,
              breakfast_included, free_cancellation, no_prepayment, rooms_left,
-             rate_plan_id, max_persons, scrape_batch_id)
+             rate_plan_id, max_persons, scrape_batch_id,
+             breakfast_text, cancel_text, payment_text)
             VALUES (:hotel_id, :rate_date, :status, :rate, :currency, :room_type,
                     :breakfast, :cancel, :prepay, :rooms_left,
-                    :rate_plan_id, :max_persons, :batch_id)
+                    :rate_plan_id, :max_persons, :batch_id,
+                    :breakfast_text, :cancel_text, :payment_text)
         """),
         {
             'hotel_id': hotel_id,
@@ -217,6 +219,9 @@ def save_rate(db: Session, rate: RateData, hotel_id: int, batch_id: uuid.UUID):
             'rate_plan_id': rate.rate_plan_id,
             'max_persons': rate.max_persons,
             'batch_id': str(batch_id),
+            'breakfast_text': rate.breakfast_text,
+            'cancel_text': rate.cancel_text,
+            'payment_text': rate.payment_text,
         }
     )
 

backend/services/scraper_backends/base.py

@@ -38,6 +38,9 @@ class RateData:
     available_qty: Optional[int] = None  # Future: from hotel page dropdown
     rate_plan_id: Optional[str] = None   # block_id from hotel page (identifies rate variant)
     max_persons: Optional[int] = None    # Occupancy this rate applies to
+    breakfast_text: Optional[str] = None # e.g. "Superb breakfast included" / "Superb breakfast £17.50"
+    cancel_text: Optional[str] = None    # e.g. "Free cancellation before 15 July 2026" / "Non-refundable"
+    payment_text: Optional[str] = None   # e.g. "No prepayment needed – pay at the property" / "Pay online"
 
 
 @dataclass

backend/services/scraper_backends/playwright_hotel_page.py

@@ -109,26 +109,61 @@ _EXTRACT_RATES_JS = """
 
         const cells = tr.querySelectorAll('td');
 
-        // Scan ALL cells for breakfast — the conditions column index varies
+        // Extract condition detail lines using the same text-search approach as the
-        // across hotel page templates (some hotels swap occupancy + conditions).
+        // original scrapy spider: grab <li> items from the row (Booking.com renders
-        // Also check data-fltrs which carries a mealplan flag on some properties.
+        // meal plan, cancellation policy and payment method each as a separate <li>),
-        const rowText = (tr.innerText || '').toLowerCase();
+        // then search each line for known keywords. No match → null, not assumed false.
-        const breakfastIncluded = rowText.includes('breakfast')
+        const liTexts = Array.from(tr.querySelectorAll('li'))
-            || fltrs.mealplan === 1
+            .map(li => (li.innerText || '').trim())
-            || fltrs.breakfast_included === 1;
+            .filter(Boolean);
+        const detailLines = liTexts.length
+            ? liTexts
+            : (tr.innerText || '').split('\\n').map(l => l.trim()).filter(l => l.length > 2 && l.length < 200);
 
-        const nonRefundable = (fltrs.non_refundable === 1);
+        let breakfastText = null;
+        let cancelText    = null;
+        let paymentText   = null;
 
-        // Cancellation text: find whichever cell mentions it.
+        for (const line of detailLines) {
-        let condCell = '';
+            const ll = line.toLowerCase();
-        for (const cell of cells) {
+            if (!breakfastText && ll.includes('breakfast')) breakfastText = line;
-            const t = cell.innerText || '';
+            if (!cancelText && (ll.includes('free cancellation') || ll.includes('non-refundable')
-            if (t.toLowerCase().includes('free cancellation')) { condCell = t; break; }
+                || ll.includes('total cost to cancel') || ll.includes('fully chargeable')
+                || ll.includes('partially refundable'))) cancelText = line;
+            if (!paymentText && (ll.includes('no prepayment') || ll.includes('pay at the property')
+                || ll.includes('pay the property') || ll.includes('pay online')
+                || ll.includes('pay now'))) paymentText = line;
         }
-        if (!condCell && cells.length >= 3) condCell = cells[2].innerText || '';
 
-        const cancelMatch = condCell.match(/free cancellation before ([\\w\\s]+)/i);
+        // Cross-check with structured data-fltrs — use as fallback when text is absent
-        const freeCancelText = cancelMatch ? cancelMatch[1].trim() : null;
+        const fltrsBreakfast    = fltrs.mealplan === 1 || fltrs.breakfast_included === 1;
+        const fltrsNonRefundable = fltrs.non_refundable === 1;
+
+        if (!breakfastText && fltrsBreakfast)     breakfastText = 'Breakfast included';
+        if (!cancelText && fltrsNonRefundable)     cancelText    = 'Non-refundable';
+        if (!cancelText && fltrs.non_refundable === 0) cancelText = 'Free cancellation';
+
+        // Derive booleans — text is authoritative, fltrs as fallback
+        let breakfastIncluded = null;
+        if (breakfastText) {
+            breakfastIncluded = breakfastText.toLowerCase().includes('included') || fltrsBreakfast;
+        } else if (fltrsBreakfast) {
+            breakfastIncluded = true;
+        }
+
+        let freeCancellation = null;
+        if (cancelText) {
+            freeCancellation = cancelText.toLowerCase().includes('free cancellation');
+        } else if (fltrsNonRefundable) {
+            freeCancellation = false;
+        }
+
+        let noPrepayment = null;
+        if (paymentText) {
+            const pll = paymentText.toLowerCase();
+            noPrepayment = pll.includes('no prepayment') || pll.includes('pay at the property')
+                        || pll.includes('pay the property');
+        }
 
         // Max persons: read from visible span text (same as original scrapy spider).
         // block_id segment 2 is NOT reliable — some hotels use 0 there regardless of occupancy.
@@ -175,9 +210,12 @@ _EXTRACT_RATES_JS = """
             avail_count:        qtyAvailable,
             block_id:           blockId,
             price:              priceRaw ? parseInt(priceRaw) : null,
+            breakfast_text:     breakfastText,
+            cancel_text:        cancelText,
+            payment_text:       paymentText,
             breakfast_included: breakfastIncluded,
-            non_refundable:     nonRefundable,
+            free_cancellation:  freeCancellation,
-            free_cancel_text:   freeCancelText,
+            no_prepayment:      noPrepayment,
             max_persons:        maxPersons,
         });
     }
@@ -376,12 +414,16 @@ class PlaywrightHotelPageBackend(ScraperBackend):
                     rate_gross=Decimal(plan['price']),
                     currency='GBP',
                     room_type=plan['room_name'] or None,
-                    breakfast_included=plan['breakfast_included'],
+                    breakfast_included=plan['breakfast_included'],  # True/False/None
-                    free_cancellation=not plan['non_refundable'],
+                    free_cancellation=plan['free_cancellation'],    # True/False/None
-                    rooms_left=plan['avail_count'],   # qty from dropdown (max 10) or scarcity text
+                    no_prepayment=plan['no_prepayment'],            # True/False/None
+                    rooms_left=plan['avail_count'],
                     available_qty=plan['avail_count'],
                     rate_plan_id=plan['block_id'] or None,
                     max_persons=plan['max_persons'],
+                    breakfast_text=plan['breakfast_text'],
+                    cancel_text=plan['cancel_text'],
+                    payment_text=plan['payment_text'],
                 ))
 
             logger.info(

frontend/src/pages/MarketView.tsx

@@ -1276,7 +1276,7 @@ const MonthSelector: React.FC<{
 interface RateHistoryPoint { t: string | null; rate: number }
 interface RateHistorySeries { room_type: string; points: RateHistoryPoint[] }
 interface RateHistoryModal { hotelId: number; hotelName: string; stayDate: string }
-interface RatePlan { meal: string; cancel: string; price: number; max_persons: number | null }
+interface RatePlan { breakfast: string | null; cancel: string | null; payment: string | null; price: number; max_persons: number | null }
 interface RoomSnapshot { room_type: string; rooms_left: number | null; availability_status: string; plans: RatePlan[] }
 interface RateSnapshot { stay_date: string; legacy: boolean; rooms: RoomSnapshot[] }
 
@@ -1350,11 +1350,25 @@ const RateHistoryModalComponent: React.FC<{ modal: RateHistoryModal; onClose: ()
                         <tbody>
                           {room.plans.map((plan, pi) => (
                             <tr key={pi} style={{ borderTop: '1px solid var(--border)' }}>
-                              <td style={{ padding: '6px 12px', color: 'var(--text-mid)' }}>{plan.meal}</td>
+                              <td style={{ padding: '6px 12px' }}>
-                              <td style={{ padding: '6px 12px', color: plan.cancel.startsWith('Free') ? '#16a34a' : 'var(--text-mid)' }}>
+                                {plan.breakfast && (
-                                {plan.cancel}
+                                  <div style={{ color: plan.breakfast.toLowerCase().includes('included') ? '#16a34a' : 'var(--text-mid)' }}>
+                                    {plan.breakfast}
+                                  </div>
+                                )}
+                                {plan.cancel && (
+                                  <div style={{ color: plan.cancel.toLowerCase().startsWith('free') ? '#16a34a' : 'var(--text-mid)' }}>
+                                    {plan.cancel}
+                                  </div>
+                                )}
+                                {plan.payment && (
+                                  <div style={{ color: 'var(--text-mid)' }}>{plan.payment}</div>
+                                )}
+                                {!plan.breakfast && !plan.cancel && !plan.payment && (
+                                  <span style={{ color: 'var(--text-light)', fontSize: 11 }}>—</span>
+                                )}
                               </td>
-                              <td style={{ padding: '6px 12px', textAlign: 'right', fontWeight: 600, color: 'var(--text-dark)' }}>
+                              <td style={{ padding: '6px 12px', textAlign: 'right', fontWeight: 600, color: 'var(--text-dark)', whiteSpace: 'nowrap' }}>
                                 £{plan.price.toLocaleString()}
                               </td>
                             </tr>