jtricerolph
ea240b3f96
Silent scraper breakage (200-with-empty-body) went unnoticed for 16 days because nothing flagged it. Now, after each daily direct scrape, a health check compares every enabled competitor's freshest captured rate against a staleness threshold (default 5 days, config: scrape_health_stale_days). State lives in a new direct_scrape_health table so we alert on transitions only — one email when a competitor goes stale, one when it recovers, never a daily repeat. Email uses the stack's shared SMTP integration via central_settings (recipient: scrape_health_alert_email, else SMTP reply_to/ from) — no new mail secrets in this app. The current stale set also drives an in-app warning banner on the Direct Rates page (GET /direct/health), with an on-demand re-check endpoint (POST /direct/health/check). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
50 lines
1.6 KiB
Python
50 lines
1.6 KiB
Python
"""
|
|
Daily job: scrape all enabled direct competitor hotels.
|
|
Hotels are scraped sequentially — the scraper enforces 10s delays per date
|
|
to avoid rate-limiting, so concurrent scraping is not beneficial.
|
|
"""
|
|
import logging
|
|
from sqlalchemy import text
|
|
|
|
from database import SyncSessionLocal
|
|
from services.direct_scraper import run_scrape
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def run_scrape_all_direct():
|
|
db = SyncSessionLocal()
|
|
try:
|
|
rows = db.execute(
|
|
text("""SELECT id, name, profile_name, params
|
|
FROM direct_competitor_hotels
|
|
WHERE scrape_enabled = true
|
|
ORDER BY id""")
|
|
).mappings().fetchall()
|
|
finally:
|
|
db.close()
|
|
|
|
if not rows:
|
|
log.info("No direct competitor hotels enabled for scraping")
|
|
return
|
|
|
|
log.info(f"Starting direct rate scrape for {len(rows)} hotels")
|
|
for hotel in rows:
|
|
try:
|
|
log.info(f"Scraping {hotel['name']} ({hotel['profile_name']})")
|
|
run_scrape(
|
|
hotel_id=hotel["id"],
|
|
profile_name=hotel["profile_name"],
|
|
params=hotel["params"],
|
|
)
|
|
except Exception as e:
|
|
log.error(f"Direct scrape failed for hotel {hotel['id']} ({hotel['name']}): {e}")
|
|
|
|
log.info("Direct rate scrape complete")
|
|
|
|
# Flag any competitor whose data has gone stale (and alert on transitions).
|
|
try:
|
|
from jobs.check_scrape_health import check_scrape_health
|
|
check_scrape_health()
|
|
except Exception as e:
|
|
log.error(f"Scrape health check failed: {e}")
|