Add Rate Monitor app — Booking.com + direct booking engine competitor rates
Combines Booking.com Playwright scraper (from forecasting), direct booking engine scraper (ported from laptop-archive/guestline-monitor), and Newbook own-hotel rates into one focused tool. Four views: Bookability, Market View (with price index badges + direct rate sub-rows), Direct Rates (per-competitor room breakdown, min-stay flags, hotel config/discovery), Rate Analysis (advance purchase curve, DOW chart, rate timeline, comparison table). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
e05054172f
50 changed files with 11860 additions and 0 deletions
295
backend/services/direct_scraper.py
Normal file
295
backend/services/direct_scraper.py
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
"""
|
||||
Direct booking engine scraper — adapted from guestline-monitor/app/scraper.py.
|
||||
Replaces SQLite per-hotel DBs with PostgreSQL via SyncSessionLocal.
|
||||
Logic (min-stay detection, discovery date sampling) is unchanged from the original.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone, date, timedelta
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import text
|
||||
|
||||
from database import SyncSessionLocal
|
||||
from services.direct_profiles import get_profile
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
REQUEST_DELAY = 10.0
|
||||
DISCOVERY_DELAY = 2.0
|
||||
|
||||
# Track running discovery scrapes: hotel_id -> status dict
|
||||
_discovery_status: dict[int, dict] = {}
|
||||
|
||||
|
||||
def _discovery_dates() -> list[str]:
|
||||
"""42 spread dates: one per DOW over 6 months, 6 weeks apart."""
|
||||
dates = []
|
||||
today = date.today()
|
||||
for week_offset in range(6):
|
||||
base = today + timedelta(weeks=week_offset * 4)
|
||||
for dow in range(7):
|
||||
days_ahead = (dow - base.weekday()) % 7
|
||||
d = base + timedelta(days=days_ahead + 7)
|
||||
iso = d.isoformat()
|
||||
if iso not in dates:
|
||||
dates.append(iso)
|
||||
return sorted(dates)
|
||||
|
||||
|
||||
async def run_discovery(hotel_id: int, profile_name: str, params: dict):
|
||||
"""Sample ~42 spread dates to find all room/rate type IDs for a competitor hotel."""
|
||||
_discovery_status[hotel_id] = {
|
||||
"state": "running", "done": 0, "total": 0,
|
||||
"found_rooms": [], "found_rates": [],
|
||||
}
|
||||
profile = get_profile(profile_name)
|
||||
dates = _discovery_dates()
|
||||
_discovery_status[hotel_id]["total"] = len(dates)
|
||||
|
||||
found_rooms: set[str] = set()
|
||||
found_rates: set[str] = set()
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
for i, arrival in enumerate(dates):
|
||||
await asyncio.sleep(DISCOVERY_DELAY)
|
||||
try:
|
||||
rooms = await profile.fetch_night_rates(client, params, arrival)
|
||||
for room in rooms:
|
||||
found_rooms.add(room["roomId"])
|
||||
found_rates.add(room["rateId"])
|
||||
except Exception as e:
|
||||
log.warning(f"Discovery hotel {hotel_id} {arrival}: {e}")
|
||||
_discovery_status[hotel_id]["done"] = i + 1
|
||||
_discovery_status[hotel_id]["found_rooms"] = sorted(found_rooms)
|
||||
_discovery_status[hotel_id]["found_rates"] = sorted(found_rates)
|
||||
|
||||
# Fetch friendly names if the profile supports it
|
||||
if hasattr(profile, "fetch_category_names"):
|
||||
try:
|
||||
cat_names, rate_names = await profile.fetch_category_names(client, params)
|
||||
_discovery_status[hotel_id]["room_labels"] = cat_names
|
||||
_discovery_status[hotel_id]["rate_labels"] = rate_names
|
||||
# Merge into DB (existing user labels win)
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
row = db.execute(
|
||||
text("SELECT room_labels, rate_labels FROM direct_competitor_hotels WHERE id = :id"),
|
||||
{"id": hotel_id}
|
||||
).mappings().fetchone()
|
||||
if row:
|
||||
import json
|
||||
existing_rooms = row["room_labels"] or {}
|
||||
existing_rates = row["rate_labels"] or {}
|
||||
merged_rooms = {**cat_names, **existing_rooms}
|
||||
merged_rates = {**rate_names, **existing_rates}
|
||||
db.execute(
|
||||
text("""UPDATE direct_competitor_hotels
|
||||
SET room_labels = :rl, rate_labels = :ratel
|
||||
WHERE id = :id"""),
|
||||
{"rl": json.dumps(merged_rooms), "ratel": json.dumps(merged_rates), "id": hotel_id}
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
log.warning(f"Discovery hotel {hotel_id}: could not fetch category names: {e}")
|
||||
|
||||
_discovery_status[hotel_id]["state"] = "complete"
|
||||
log.info(f"Discovery complete hotel {hotel_id}: {len(found_rooms)} rooms, {len(found_rates)} rates")
|
||||
|
||||
|
||||
def get_discovery_status(hotel_id: int) -> dict:
|
||||
return _discovery_status.get(hotel_id, {"state": "idle"})
|
||||
|
||||
|
||||
def run_scrape(hotel_id: int, profile_name: str, params: dict):
|
||||
"""Full scrape run for one configured hotel. Writes to direct_rates + direct_scrape_runs."""
|
||||
scraped_at = datetime.now(timezone.utc)
|
||||
log.info(f"Direct scrape started for hotel {hotel_id}")
|
||||
profile = get_profile(profile_name)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
db.execute(
|
||||
text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"),
|
||||
{"ts": scraped_at, "id": hotel_id}
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
arrival_dates = await profile.fetch_arrival_dates(client, params)
|
||||
except Exception as e:
|
||||
log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}")
|
||||
return
|
||||
|
||||
log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates")
|
||||
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
result = db.execute(
|
||||
text("INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found) VALUES (:hid, :ts, :df) RETURNING id"),
|
||||
{"hid": hotel_id, "ts": scraped_at, "df": len(arrival_dates)}
|
||||
)
|
||||
run_id = result.fetchone()[0]
|
||||
db.commit()
|
||||
|
||||
prev_dates = {r[0].isoformat() for r in db.execute(
|
||||
text("SELECT DISTINCT stay_date FROM direct_rates WHERE hotel_id = :hid AND stay_date >= :today"),
|
||||
{"hid": hotel_id, "today": date.today()}
|
||||
).fetchall()}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
arrival_set = set(arrival_dates)
|
||||
missing_dates = sorted(prev_dates - arrival_set)
|
||||
if missing_dates:
|
||||
log.info(f"Hotel {hotel_id}: {len(missing_dates)} dates absent, checking min-stay")
|
||||
|
||||
# Min-stay check for missing dates
|
||||
for fd in missing_dates:
|
||||
fd_date = date.fromisoformat(fd)
|
||||
windows = [
|
||||
(fd_date - timedelta(days=1), fd_date + timedelta(days=1)),
|
||||
(fd_date, fd_date + timedelta(days=2)),
|
||||
]
|
||||
min_stay_rooms = None
|
||||
min_stay_other_night = None
|
||||
|
||||
await asyncio.sleep(REQUEST_DELAY)
|
||||
for win_start, win_end in windows:
|
||||
try:
|
||||
rooms_2n = await profile.fetch_night_rates(client, params, win_start.isoformat(), nights=2)
|
||||
if rooms_2n:
|
||||
min_stay_rooms = rooms_2n
|
||||
companion = win_start if win_start.isoformat() != fd else (win_start + timedelta(days=1))
|
||||
min_stay_other_night = companion.isoformat()
|
||||
log.info(f" Hotel {hotel_id} {fd}: min-stay detected")
|
||||
break
|
||||
except Exception as e:
|
||||
log.warning(f" Hotel {hotel_id} {fd}: min-stay check {win_start}: {e}")
|
||||
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
last_rows = db.execute(
|
||||
text("""SELECT DISTINCT ON (room_id, rate_id)
|
||||
room_id, rate_id, availability, price_excl, price_incl, currency
|
||||
FROM direct_rates
|
||||
WHERE hotel_id = :hid AND stay_date = :fd
|
||||
ORDER BY room_id, rate_id, scraped_at DESC"""),
|
||||
{"hid": hotel_id, "fd": fd}
|
||||
).mappings().fetchall()
|
||||
|
||||
insert_rows = []
|
||||
for r in last_rows:
|
||||
if min_stay_rooms is not None:
|
||||
match = next(
|
||||
(m for m in min_stay_rooms
|
||||
if m["roomId"] == r["room_id"] and m["rateId"] == r["rate_id"]
|
||||
and m.get("prices")),
|
||||
None
|
||||
)
|
||||
if match:
|
||||
prices = match["prices"]
|
||||
win_start_used = windows[0][0] if min_stay_other_night == windows[0][0].isoformat() else windows[1][0]
|
||||
idx = 1 if win_start_used.isoformat() != fd else 0
|
||||
if len(prices) > idx:
|
||||
p = prices[idx]
|
||||
insert_rows.append({
|
||||
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
|
||||
"room_id": r["room_id"], "rate_id": r["rate_id"],
|
||||
"avail": match.get("availability", r["availability"]),
|
||||
"pe": p["amountBeforeTax"], "pi": p["amountAfterTax"],
|
||||
"cur": match.get("currencyCode", r["currency"]), "ms": 2
|
||||
})
|
||||
continue
|
||||
insert_rows.append({
|
||||
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
|
||||
"room_id": r["room_id"], "rate_id": r["rate_id"],
|
||||
"avail": r["availability"], "pe": r["price_excl"], "pi": r["price_incl"],
|
||||
"cur": r["currency"], "ms": 2
|
||||
})
|
||||
else:
|
||||
insert_rows.append({
|
||||
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
|
||||
"room_id": r["room_id"], "rate_id": r["rate_id"],
|
||||
"avail": 0, "pe": r["price_excl"], "pi": r["price_incl"],
|
||||
"cur": r["currency"], "ms": None
|
||||
})
|
||||
|
||||
if insert_rows:
|
||||
db.execute(
|
||||
text("""INSERT INTO direct_rates
|
||||
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
|
||||
availability, price_excl, price_incl, currency, min_stay_nights)
|
||||
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
|
||||
:avail, :pe, :pi, :cur, :ms)"""),
|
||||
insert_rows
|
||||
)
|
||||
db.commit()
|
||||
status = "min-stay(2N)" if min_stay_rooms else "fully-booked"
|
||||
log.info(f" Hotel {hotel_id} {fd}: recorded as {status}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Scrape all bookable dates
|
||||
rows_saved = 0
|
||||
for i, arrival in enumerate(arrival_dates):
|
||||
await asyncio.sleep(REQUEST_DELAY)
|
||||
try:
|
||||
rooms = await profile.fetch_night_rates(client, params, arrival)
|
||||
if not rooms:
|
||||
continue
|
||||
insert_rows = [
|
||||
{
|
||||
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": arrival,
|
||||
"room_id": room["roomId"], "rate_id": room["rateId"],
|
||||
"avail": room["availability"],
|
||||
"pe": room["prices"][0]["amountBeforeTax"],
|
||||
"pi": room["prices"][0]["amountAfterTax"],
|
||||
"cur": room.get("currencyCode", "GBP"),
|
||||
"ms": room.get("min_stay_nights")
|
||||
}
|
||||
for room in rooms if room.get("prices")
|
||||
]
|
||||
if insert_rows:
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
db.execute(
|
||||
text("""INSERT INTO direct_rates
|
||||
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
|
||||
availability, price_excl, price_incl, currency, min_stay_nights)
|
||||
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
|
||||
:avail, :pe, :pi, :cur, :ms)"""),
|
||||
insert_rows
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
rows_saved += len(insert_rows)
|
||||
log.info(f" Hotel {hotel_id} {arrival}: {len(rooms)} combos [{i+1}/{len(arrival_dates)}]")
|
||||
except Exception as e:
|
||||
log.error(f" Hotel {hotel_id} {arrival}: ERROR - {e}")
|
||||
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
db.execute(
|
||||
text("UPDATE direct_scrape_runs SET rows_saved = :rs WHERE id = :id"),
|
||||
{"rs": rows_saved, "id": run_id}
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved")
|
||||
Loading…
Add table
Add a link
Reference in a new issue