jtricerolph
e05054172f
Combines Booking.com Playwright scraper (from forecasting), direct booking engine scraper (ported from laptop-archive/guestline-monitor), and Newbook own-hotel rates into one focused tool. Four views: Bookability, Market View (with price index badges + direct rate sub-rows), Direct Rates (per-competitor room breakdown, min-stay flags, hotel config/discovery), Rate Analysis (advance purchase curve, DOW chart, rate timeline, comparison table). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
43 lines
1.3 KiB
Python
43 lines
1.3 KiB
Python
"""
|
|
Daily job: scrape all enabled direct competitor hotels.
|
|
Hotels are scraped sequentially — the scraper enforces 10s delays per date
|
|
to avoid rate-limiting, so concurrent scraping is not beneficial.
|
|
"""
|
|
import logging
|
|
from sqlalchemy import text
|
|
|
|
from database import SyncSessionLocal
|
|
from services.direct_scraper import run_scrape
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def run_scrape_all_direct():
|
|
db = SyncSessionLocal()
|
|
try:
|
|
rows = db.execute(
|
|
text("""SELECT id, name, profile_name, params
|
|
FROM direct_competitor_hotels
|
|
WHERE scrape_enabled = true
|
|
ORDER BY id""")
|
|
).mappings().fetchall()
|
|
finally:
|
|
db.close()
|
|
|
|
if not rows:
|
|
log.info("No direct competitor hotels enabled for scraping")
|
|
return
|
|
|
|
log.info(f"Starting direct rate scrape for {len(rows)} hotels")
|
|
for hotel in rows:
|
|
try:
|
|
log.info(f"Scraping {hotel['name']} ({hotel['profile_name']})")
|
|
run_scrape(
|
|
hotel_id=hotel["id"],
|
|
profile_name=hotel["profile_name"],
|
|
params=hotel["params"],
|
|
)
|
|
except Exception as e:
|
|
log.error(f"Direct scrape failed for hotel {hotel['id']} ({hotel['name']}): {e}")
|
|
|
|
log.info("Direct rate scrape complete")
|