jtricerolph
1dc8c0a945
Proxy config, DataImpulse sticky-session username syntax, and Playwright/ httpx proxy builders now live in one place (services/proxy.py) instead of being duplicated across the Booking.com backend and the /config/proxy test endpoint. Both scrapers consume it. - services/proxy.py: load_config/normalize (DB-authoritative, env fallback), new_session_id, username, playwright_proxy, httpx_proxy_url - PlaywrightLocalBackend delegates proxy building to the module - get_scraper_backend factory uses proxy.load_config (one resolution path) - test_proxy_config endpoint uses the shared URL builder; httpx proxies= -> proxy= (forward-compatible, 0.28-safe) - Direct booking-engine scraper (httpx) can now route through the same proxy, gated by the direct_scraper_use_proxy flag (default off, plumbing ready) Dead code removed: set_scraper_paused (never called — rotate-on-block replaced pause-on-block), get_competitor_matrix / get_hotels_list / update_hotel_tier (endpoints have their own SQL), unused PROXY_KEYS tuple. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
307 lines
13 KiB
Python
307 lines
13 KiB
Python
"""
|
|
Direct booking engine scraper — adapted from guestline-monitor/app/scraper.py.
|
|
Replaces SQLite per-hotel DBs with PostgreSQL via SyncSessionLocal.
|
|
Logic (min-stay detection, discovery date sampling) is unchanged from the original.
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone, date, timedelta
|
|
|
|
import httpx
|
|
from sqlalchemy import text
|
|
|
|
from database import SyncSessionLocal
|
|
from services import proxy as proxy_util
|
|
from services.direct_profiles import get_profile
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
REQUEST_DELAY = 10.0
|
|
DISCOVERY_DELAY = 2.0
|
|
|
|
|
|
def _direct_proxy_url() -> str | None:
|
|
"""Proxy URL for direct-engine scraping, or None. Shares the Booking.com
|
|
proxy config but is gated by the `direct_scraper_use_proxy` flag (default
|
|
off — booking-engine APIs usually don't need it). Ready to switch on."""
|
|
db = SyncSessionLocal()
|
|
try:
|
|
return proxy_util.direct_httpx_proxy(db)
|
|
finally:
|
|
db.close()
|
|
|
|
# Track running discovery scrapes: hotel_id -> status dict
|
|
_discovery_status: dict[int, dict] = {}
|
|
|
|
|
|
def _discovery_dates() -> list[str]:
|
|
"""42 spread dates: one per DOW over 6 months, 6 weeks apart."""
|
|
dates = []
|
|
today = date.today()
|
|
for week_offset in range(6):
|
|
base = today + timedelta(weeks=week_offset * 4)
|
|
for dow in range(7):
|
|
days_ahead = (dow - base.weekday()) % 7
|
|
d = base + timedelta(days=days_ahead + 7)
|
|
iso = d.isoformat()
|
|
if iso not in dates:
|
|
dates.append(iso)
|
|
return sorted(dates)
|
|
|
|
|
|
async def run_discovery(hotel_id: int, profile_name: str, params: dict):
|
|
"""Sample ~42 spread dates to find all room/rate type IDs for a competitor hotel."""
|
|
_discovery_status[hotel_id] = {
|
|
"state": "running", "done": 0, "total": 0,
|
|
"found_rooms": [], "found_rates": [],
|
|
}
|
|
profile = get_profile(profile_name)
|
|
dates = _discovery_dates()
|
|
_discovery_status[hotel_id]["total"] = len(dates)
|
|
|
|
found_rooms: set[str] = set()
|
|
found_rates: set[str] = set()
|
|
|
|
async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client:
|
|
for i, arrival in enumerate(dates):
|
|
await asyncio.sleep(DISCOVERY_DELAY)
|
|
try:
|
|
rooms = await profile.fetch_night_rates(client, params, arrival)
|
|
for room in rooms:
|
|
found_rooms.add(room["roomId"])
|
|
found_rates.add(room["rateId"])
|
|
except Exception as e:
|
|
log.warning(f"Discovery hotel {hotel_id} {arrival}: {e}")
|
|
_discovery_status[hotel_id]["done"] = i + 1
|
|
_discovery_status[hotel_id]["found_rooms"] = sorted(found_rooms)
|
|
_discovery_status[hotel_id]["found_rates"] = sorted(found_rates)
|
|
|
|
# Fetch friendly names if the profile supports it
|
|
if hasattr(profile, "fetch_category_names"):
|
|
try:
|
|
cat_names, rate_names = await profile.fetch_category_names(client, params)
|
|
_discovery_status[hotel_id]["room_labels"] = cat_names
|
|
_discovery_status[hotel_id]["rate_labels"] = rate_names
|
|
# Merge into DB (existing user labels win)
|
|
db = SyncSessionLocal()
|
|
try:
|
|
row = db.execute(
|
|
text("SELECT room_labels, rate_labels FROM direct_competitor_hotels WHERE id = :id"),
|
|
{"id": hotel_id}
|
|
).mappings().fetchone()
|
|
if row:
|
|
import json
|
|
existing_rooms = row["room_labels"] or {}
|
|
existing_rates = row["rate_labels"] or {}
|
|
merged_rooms = {**cat_names, **existing_rooms}
|
|
merged_rates = {**rate_names, **existing_rates}
|
|
db.execute(
|
|
text("""UPDATE direct_competitor_hotels
|
|
SET room_labels = :rl, rate_labels = :ratel
|
|
WHERE id = :id"""),
|
|
{"rl": json.dumps(merged_rooms), "ratel": json.dumps(merged_rates), "id": hotel_id}
|
|
)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
except Exception as e:
|
|
log.warning(f"Discovery hotel {hotel_id}: could not fetch category names: {e}")
|
|
|
|
_discovery_status[hotel_id]["state"] = "complete"
|
|
log.info(f"Discovery complete hotel {hotel_id}: {len(found_rooms)} rooms, {len(found_rates)} rates")
|
|
|
|
|
|
def get_discovery_status(hotel_id: int) -> dict:
|
|
return _discovery_status.get(hotel_id, {"state": "idle"})
|
|
|
|
|
|
def run_scrape(hotel_id: int, profile_name: str, params: dict):
|
|
"""Full scrape run for one configured hotel. Writes to direct_rates + direct_scrape_runs."""
|
|
scraped_at = datetime.now(timezone.utc)
|
|
log.info(f"Direct scrape started for hotel {hotel_id}")
|
|
profile = get_profile(profile_name)
|
|
|
|
loop = asyncio.new_event_loop()
|
|
try:
|
|
loop.run_until_complete(_run_scrape_async(hotel_id, profile, params, scraped_at))
|
|
finally:
|
|
loop.close()
|
|
|
|
db = SyncSessionLocal()
|
|
try:
|
|
db.execute(
|
|
text("UPDATE direct_competitor_hotels SET last_scraped_at = :ts WHERE id = :id"),
|
|
{"ts": scraped_at, "id": hotel_id}
|
|
)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
async def _run_scrape_async(hotel_id: int, profile, params: dict, scraped_at: datetime):
|
|
async with httpx.AsyncClient(proxy=_direct_proxy_url()) as client:
|
|
try:
|
|
arrival_dates = await profile.fetch_arrival_dates(client, params)
|
|
except Exception as e:
|
|
log.error(f"Hotel {hotel_id}: failed to fetch arrival dates: {e}")
|
|
return
|
|
|
|
log.info(f"Hotel {hotel_id}: {len(arrival_dates)} bookable dates")
|
|
|
|
db = SyncSessionLocal()
|
|
try:
|
|
result = db.execute(
|
|
text("INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found) VALUES (:hid, :ts, :df) RETURNING id"),
|
|
{"hid": hotel_id, "ts": scraped_at, "df": len(arrival_dates)}
|
|
)
|
|
run_id = result.fetchone()[0]
|
|
db.commit()
|
|
|
|
prev_dates = {r[0].isoformat() for r in db.execute(
|
|
text("SELECT DISTINCT stay_date FROM direct_rates WHERE hotel_id = :hid AND stay_date >= :today"),
|
|
{"hid": hotel_id, "today": date.today()}
|
|
).fetchall()}
|
|
finally:
|
|
db.close()
|
|
|
|
arrival_set = set(arrival_dates)
|
|
missing_dates = sorted(prev_dates - arrival_set)
|
|
if missing_dates:
|
|
log.info(f"Hotel {hotel_id}: {len(missing_dates)} dates absent, checking min-stay")
|
|
|
|
# Min-stay check for missing dates
|
|
for fd in missing_dates:
|
|
fd_date = date.fromisoformat(fd)
|
|
windows = [
|
|
(fd_date - timedelta(days=1), fd_date + timedelta(days=1)),
|
|
(fd_date, fd_date + timedelta(days=2)),
|
|
]
|
|
min_stay_rooms = None
|
|
min_stay_other_night = None
|
|
|
|
await asyncio.sleep(REQUEST_DELAY)
|
|
for win_start, win_end in windows:
|
|
try:
|
|
rooms_2n = await profile.fetch_night_rates(client, params, win_start.isoformat(), nights=2)
|
|
if rooms_2n:
|
|
min_stay_rooms = rooms_2n
|
|
companion = win_start if win_start.isoformat() != fd else (win_start + timedelta(days=1))
|
|
min_stay_other_night = companion.isoformat()
|
|
log.info(f" Hotel {hotel_id} {fd}: min-stay detected")
|
|
break
|
|
except Exception as e:
|
|
log.warning(f" Hotel {hotel_id} {fd}: min-stay check {win_start}: {e}")
|
|
|
|
db = SyncSessionLocal()
|
|
try:
|
|
last_rows = db.execute(
|
|
text("""SELECT DISTINCT ON (room_id, rate_id)
|
|
room_id, rate_id, availability, price_excl, price_incl, currency
|
|
FROM direct_rates
|
|
WHERE hotel_id = :hid AND stay_date = :fd
|
|
ORDER BY room_id, rate_id, scraped_at DESC"""),
|
|
{"hid": hotel_id, "fd": fd}
|
|
).mappings().fetchall()
|
|
|
|
insert_rows = []
|
|
for r in last_rows:
|
|
if min_stay_rooms is not None:
|
|
match = next(
|
|
(m for m in min_stay_rooms
|
|
if m["roomId"] == r["room_id"] and m["rateId"] == r["rate_id"]
|
|
and m.get("prices")),
|
|
None
|
|
)
|
|
if match:
|
|
prices = match["prices"]
|
|
win_start_used = windows[0][0] if min_stay_other_night == windows[0][0].isoformat() else windows[1][0]
|
|
idx = 1 if win_start_used.isoformat() != fd else 0
|
|
if len(prices) > idx:
|
|
p = prices[idx]
|
|
insert_rows.append({
|
|
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
|
|
"room_id": r["room_id"], "rate_id": r["rate_id"],
|
|
"avail": match.get("availability", r["availability"]),
|
|
"pe": p["amountBeforeTax"], "pi": p["amountAfterTax"],
|
|
"cur": match.get("currencyCode", r["currency"]), "ms": 2
|
|
})
|
|
continue
|
|
insert_rows.append({
|
|
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
|
|
"room_id": r["room_id"], "rate_id": r["rate_id"],
|
|
"avail": r["availability"], "pe": r["price_excl"], "pi": r["price_incl"],
|
|
"cur": r["currency"], "ms": 2
|
|
})
|
|
else:
|
|
insert_rows.append({
|
|
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": fd,
|
|
"room_id": r["room_id"], "rate_id": r["rate_id"],
|
|
"avail": 0, "pe": r["price_excl"], "pi": r["price_incl"],
|
|
"cur": r["currency"], "ms": None
|
|
})
|
|
|
|
if insert_rows:
|
|
db.execute(
|
|
text("""INSERT INTO direct_rates
|
|
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
|
|
availability, price_excl, price_incl, currency, min_stay_nights)
|
|
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
|
|
:avail, :pe, :pi, :cur, :ms)"""),
|
|
insert_rows
|
|
)
|
|
db.commit()
|
|
status = "min-stay(2N)" if min_stay_rooms else "fully-booked"
|
|
log.info(f" Hotel {hotel_id} {fd}: recorded as {status}")
|
|
finally:
|
|
db.close()
|
|
|
|
# Scrape all bookable dates
|
|
rows_saved = 0
|
|
for i, arrival in enumerate(arrival_dates):
|
|
await asyncio.sleep(REQUEST_DELAY)
|
|
try:
|
|
rooms = await profile.fetch_night_rates(client, params, arrival)
|
|
if not rooms:
|
|
continue
|
|
insert_rows = [
|
|
{
|
|
"hid": hotel_id, "run_id": run_id, "ts": scraped_at, "sd": arrival,
|
|
"room_id": room["roomId"], "rate_id": room["rateId"],
|
|
"avail": room["availability"],
|
|
"pe": room["prices"][0]["amountBeforeTax"],
|
|
"pi": room["prices"][0]["amountAfterTax"],
|
|
"cur": room.get("currencyCode", "GBP"),
|
|
"ms": room.get("min_stay_nights")
|
|
}
|
|
for room in rooms if room.get("prices")
|
|
]
|
|
if insert_rows:
|
|
db = SyncSessionLocal()
|
|
try:
|
|
db.execute(
|
|
text("""INSERT INTO direct_rates
|
|
(hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
|
|
availability, price_excl, price_incl, currency, min_stay_nights)
|
|
VALUES (:hid, :run_id, :ts, :sd, :room_id, :rate_id,
|
|
:avail, :pe, :pi, :cur, :ms)"""),
|
|
insert_rows
|
|
)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
rows_saved += len(insert_rows)
|
|
log.info(f" Hotel {hotel_id} {arrival}: {len(rooms)} combos [{i+1}/{len(arrival_dates)}]")
|
|
except Exception as e:
|
|
log.error(f" Hotel {hotel_id} {arrival}: ERROR - {e}")
|
|
|
|
db = SyncSessionLocal()
|
|
try:
|
|
db.execute(
|
|
text("UPDATE direct_scrape_runs SET rows_saved = :rs WHERE id = :id"),
|
|
{"rs": rows_saved, "id": run_id}
|
|
)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
log.info(f"Hotel {hotel_id}: scrape complete, {rows_saved} rows saved")
|