From b916f1a09b3558c9b49a7eab93809771d539e9e3 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 13 Aug 2026 14:09:13 +0000 Subject: [PATCH] Fix Guestline + Mews direct scrapers after engine API changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both competitor booking engines changed their API contracts, silently breaking their direct scrapes ~16 days ago (200 responses with empty bodies, so no exception was ever raised): - Guestline: the /enhanced availability endpoint now returns a bare [] for every date. Switch to the base /api/availabilities/{coll}/{hotel} endpoint, which returns {"rooms":[...]} with the same room shape. - Mews: getAvailability now rejects the old body with "Invalid EnterpriseId" — it requires enterpriseId + serviceId. getPricing additionally requires currencyCode. Capture the enterprise's defaultCurrencyCode in _ensure_config and send all three. Also stop last_scraped_at advancing when a run saves 0 rows, so the overview no longer reports a phantom-fresh scrape while the per-date data stays stale — the symptom that surfaced this. Verified live: Three Ways now yields 41 rows/night, Old Stocks 10. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/services/direct_profiles/guestline.py | 4 +-- backend/services/direct_profiles/mews.py | 12 +++++++- backend/services/direct_scraper.py | 29 ++++++++++++------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/backend/services/direct_profiles/guestline.py b/backend/services/direct_profiles/guestline.py index bade8a4..9e88b16 100644 --- a/backend/services/direct_profiles/guestline.py +++ b/backend/services/direct_profiles/guestline.py @@ -39,11 +39,11 @@ class GuestlineProfile(BaseProfile): hotel_id = params["hotel_id"] collection_id = params.get("collection_id", "MT") dep = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat() - url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}/enhanced" + url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}" r = await client.get(url, params={ "arrival": arrival, "departure": dep, "adults": 2, "children": 0, }, headers=HEADERS, timeout=20) if r.status_code == 404: return [] r.raise_for_status() - return r.json().get("availabilities", {}).get("rooms", []) + return r.json().get("rooms", []) diff --git a/backend/services/direct_profiles/mews.py b/backend/services/direct_profiles/mews.py index d495060..00d7977 100644 --- a/backend/services/direct_profiles/mews.py +++ b/backend/services/direct_profiles/mews.py @@ -58,7 +58,11 @@ async def _ensure_config(client, params: dict) -> bool: if not enterprise_id: enterprise_id = (data.get("enterprises") or [{}])[0].get("id", "") - tz_name = (data.get("enterprises") or [{}])[0].get("ianaTimeZoneIdentifier", "Europe/London") + enterprises = data.get("enterprises") or [] + ent_obj = next((e for e in enterprises if e.get("id") == enterprise_id), + enterprises[0] if enterprises else {}) + tz_name = ent_obj.get("ianaTimeZoneIdentifier", "Europe/London") + currency = ent_obj.get("defaultCurrencyCode") or "GBP" # Adult age category: classification=="Adult" or no maximalAge, filtered to this service age_cat_id = "" @@ -72,6 +76,7 @@ async def _ensure_config(client, params: dict) -> bool: params["enterprise_id"] = enterprise_id params["service_id"] = service_id params["age_category_id"] = age_cat_id + params["currency"] = currency params["_tz"] = ZoneInfo(tz_name) return bool(enterprise_id and service_id) @@ -134,6 +139,8 @@ class MewsProfile(BaseProfile): f"{API_BASE}/services/getAvailability", json={ "bookingEngineId": params["booking_engine_id"], + "enterpriseId": params["enterprise_id"], + "serviceId": params["service_id"], "startUtc": _midnight_utc(today, tz), "endUtc": _midnight_utc(today + timedelta(days=DAYS_AHEAD), tz), "client": CLIENT, @@ -169,6 +176,9 @@ class MewsProfile(BaseProfile): f"{API_BASE}/services/getPricing", json={ "configurationId": params["booking_engine_id"], + "enterpriseId": params["enterprise_id"], + "serviceId": params["service_id"], + "currencyCode": params.get("currency", "GBP"), "startUtc": _midnight_utc(arrival_date, tz), "endUtc": _midnight_utc(arrival_date + timedelta(days=nights), tz), "occupancyData": [{"ageCategoryId": age_cat_id, "personCount": 2}], diff --git a/backend/services/direct_scraper.py b/backend/services/direct_scraper.py index 1687c35..c99affa 100644 --- a/backend/services/direct_scraper.py +++ b/backend/services/direct_scraper.py @@ -123,19 +123,25 @@ def run_scrape(hotel_id: int, profile_name: str, params: dict): loop = asyncio.new_event_loop() try: - loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at)) + rows_saved = loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at)) finally: loop.close() - db = SyncSessionLocal() - try: - db.execute( - text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"), - {"ts": scraped_at, "id": hotel_id} - ) - db.commit() - finally: - db.close() + # Only advance last_scraped_at when the run actually captured data. A run that + # saved nothing means the engine returned empty/errored — don't let the overview + # report a phantom-fresh scrape while the per-date data stays stale. + if rows_saved: + db = SyncSessionLocal() + try: + db.execute( + text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"), + {"ts": scraped_at, "id": hotel_id} + ) + db.commit() + finally: + db.close() + else: + log.warning(f"Hotel {hotel_id}: scrape saved 0 rows — last_scraped_at left unchanged") async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime): @@ -144,7 +150,7 @@ async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: da arrival_dates = await profile.fetch_arrival_dates(client, params) except Exception as e: log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}") - return + return 0 log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates") @@ -305,3 +311,4 @@ async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: da db.close() log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved") + return rows_saved