Fix Guestline + Mews direct scrapers after engine API changes
Both competitor booking engines changed their API contracts, silently
breaking their direct scrapes ~16 days ago (200 responses with empty
bodies, so no exception was ever raised):
- Guestline: the /enhanced availability endpoint now returns a bare []
for every date. Switch to the base /api/availabilities/{coll}/{hotel}
endpoint, which returns {"rooms":[...]} with the same room shape.
- Mews: getAvailability now rejects the old body with "Invalid
EnterpriseId" — it requires enterpriseId + serviceId. getPricing
additionally requires currencyCode. Capture the enterprise's
defaultCurrencyCode in _ensure_config and send all three.
Also stop last_scraped_at advancing when a run saves 0 rows, so the
overview no longer reports a phantom-fresh scrape while the per-date
data stays stale — the symptom that surfaced this.
Verified live: Three Ways now yields 41 rows/night, Old Stocks 10.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
22e3670531
commit
b916f1a09b
3 changed files with 31 additions and 14 deletions
|
|
@ -39,11 +39,11 @@ class GuestlineProfile(BaseProfile):
|
||||||
hotel_id = params["hotel_id"]
|
hotel_id = params["hotel_id"]
|
||||||
collection_id = params.get("collection_id", "MT")
|
collection_id = params.get("collection_id", "MT")
|
||||||
dep = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
|
dep = (date.fromisoformat(arrival) + timedelta(days=nights)).isoformat()
|
||||||
url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}/enhanced"
|
url = f"https://booking.eu.guestline.app/api/availabilities/{collection_id}/{hotel_id}"
|
||||||
r = await client.get(url, params={
|
r = await client.get(url, params={
|
||||||
"arrival": arrival, "departure": dep, "adults": 2, "children": 0,
|
"arrival": arrival, "departure": dep, "adults": 2, "children": 0,
|
||||||
}, headers=HEADERS, timeout=20)
|
}, headers=HEADERS, timeout=20)
|
||||||
if r.status_code == 404:
|
if r.status_code == 404:
|
||||||
return []
|
return []
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
return r.json().get("availabilities", {}).get("rooms", [])
|
return r.json().get("rooms", [])
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,11 @@ async def _ensure_config(client, params: dict) -> bool:
|
||||||
if not enterprise_id:
|
if not enterprise_id:
|
||||||
enterprise_id = (data.get("enterprises") or [{}])[0].get("id", "")
|
enterprise_id = (data.get("enterprises") or [{}])[0].get("id", "")
|
||||||
|
|
||||||
tz_name = (data.get("enterprises") or [{}])[0].get("ianaTimeZoneIdentifier", "Europe/London")
|
enterprises = data.get("enterprises") or []
|
||||||
|
ent_obj = next((e for e in enterprises if e.get("id") == enterprise_id),
|
||||||
|
enterprises[0] if enterprises else {})
|
||||||
|
tz_name = ent_obj.get("ianaTimeZoneIdentifier", "Europe/London")
|
||||||
|
currency = ent_obj.get("defaultCurrencyCode") or "GBP"
|
||||||
|
|
||||||
# Adult age category: classification=="Adult" or no maximalAge, filtered to this service
|
# Adult age category: classification=="Adult" or no maximalAge, filtered to this service
|
||||||
age_cat_id = ""
|
age_cat_id = ""
|
||||||
|
|
@ -72,6 +76,7 @@ async def _ensure_config(client, params: dict) -> bool:
|
||||||
params["enterprise_id"] = enterprise_id
|
params["enterprise_id"] = enterprise_id
|
||||||
params["service_id"] = service_id
|
params["service_id"] = service_id
|
||||||
params["age_category_id"] = age_cat_id
|
params["age_category_id"] = age_cat_id
|
||||||
|
params["currency"] = currency
|
||||||
params["_tz"] = ZoneInfo(tz_name)
|
params["_tz"] = ZoneInfo(tz_name)
|
||||||
return bool(enterprise_id and service_id)
|
return bool(enterprise_id and service_id)
|
||||||
|
|
||||||
|
|
@ -134,6 +139,8 @@ class MewsProfile(BaseProfile):
|
||||||
f"{API_BASE}/services/getAvailability",
|
f"{API_BASE}/services/getAvailability",
|
||||||
json={
|
json={
|
||||||
"bookingEngineId": params["booking_engine_id"],
|
"bookingEngineId": params["booking_engine_id"],
|
||||||
|
"enterpriseId": params["enterprise_id"],
|
||||||
|
"serviceId": params["service_id"],
|
||||||
"startUtc": _midnight_utc(today, tz),
|
"startUtc": _midnight_utc(today, tz),
|
||||||
"endUtc": _midnight_utc(today + timedelta(days=DAYS_AHEAD), tz),
|
"endUtc": _midnight_utc(today + timedelta(days=DAYS_AHEAD), tz),
|
||||||
"client": CLIENT,
|
"client": CLIENT,
|
||||||
|
|
@ -169,6 +176,9 @@ class MewsProfile(BaseProfile):
|
||||||
f"{API_BASE}/services/getPricing",
|
f"{API_BASE}/services/getPricing",
|
||||||
json={
|
json={
|
||||||
"configurationId": params["booking_engine_id"],
|
"configurationId": params["booking_engine_id"],
|
||||||
|
"enterpriseId": params["enterprise_id"],
|
||||||
|
"serviceId": params["service_id"],
|
||||||
|
"currencyCode": params.get("currency", "GBP"),
|
||||||
"startUtc": _midnight_utc(arrival_date, tz),
|
"startUtc": _midnight_utc(arrival_date, tz),
|
||||||
"endUtc": _midnight_utc(arrival_date + timedelta(days=nights), tz),
|
"endUtc": _midnight_utc(arrival_date + timedelta(days=nights), tz),
|
||||||
"occupancyData": [{"ageCategoryId": age_cat_id, "personCount": 2}],
|
"occupancyData": [{"ageCategoryId": age_cat_id, "personCount": 2}],
|
||||||
|
|
|
||||||
|
|
@ -123,10 +123,14 @@ def run_scrape(hotel_id: int, profile_name: str, params: dict):
|
||||||
|
|
||||||
loop = asyncio.new_event_loop()
|
loop = asyncio.new_event_loop()
|
||||||
try:
|
try:
|
||||||
loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at))
|
rows_saved = loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at))
|
||||||
finally:
|
finally:
|
||||||
loop.close()
|
loop.close()
|
||||||
|
|
||||||
|
# Only advance last_scraped_at when the run actually captured data. A run that
|
||||||
|
# saved nothing means the engine returned empty/errored — don't let the overview
|
||||||
|
# report a phantom-fresh scrape while the per-date data stays stale.
|
||||||
|
if rows_saved:
|
||||||
db = SyncSessionLocal()
|
db = SyncSessionLocal()
|
||||||
try:
|
try:
|
||||||
db.execute(
|
db.execute(
|
||||||
|
|
@ -136,6 +140,8 @@ def run_scrape(hotel_id: int, profile_name: str, params: dict):
|
||||||
db.commit()
|
db.commit()
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
else:
|
||||||
|
log.warning(f"Hotel {hotel_id}: scrape saved 0 rows — last_scraped_at left unchanged")
|
||||||
|
|
||||||
|
|
||||||
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
|
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
|
||||||
|
|
@ -144,7 +150,7 @@ async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: da
|
||||||
arrival_dates = await profile.fetch_arrival_dates(client, params)
|
arrival_dates = await profile.fetch_arrival_dates(client, params)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}")
|
log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}")
|
||||||
return
|
return 0
|
||||||
|
|
||||||
log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates")
|
log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates")
|
||||||
|
|
||||||
|
|
@ -305,3 +311,4 @@ async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: da
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved")
|
log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved")
|
||||||
|
return rows_saved
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue