Add stale-scrape health alerting (email + in-app banner)
Silent scraper breakage (200-with-empty-body) went unnoticed for 16 days because nothing flagged it. Now, after each daily direct scrape, a health check compares every enabled competitor's freshest captured rate against a staleness threshold (default 5 days, config: scrape_health_stale_days). State lives in a new direct_scrape_health table so we alert on transitions only — one email when a competitor goes stale, one when it recovers, never a daily repeat. Email uses the stack's shared SMTP integration via central_settings (recipient: scrape_health_alert_email, else SMTP reply_to/ from) — no new mail secrets in this app. The current stale set also drives an in-app warning banner on the Direct Rates page (GET /direct/health), with an on-demand re-check endpoint (POST /direct/health/check). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b916f1a09b
commit
ea240b3f96
6 changed files with 261 additions and 1 deletions
|
|
@ -121,6 +121,31 @@ async def list_hotels(user=Depends(get_current_user)):
|
|||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def scrape_health(user=Depends(get_current_user)):
|
||||
"""Competitor hotels currently flagged as stale — drives the Direct Rates banner."""
|
||||
require_cap(user, "view_direct_rates")
|
||||
async with AsyncSessionLocal() as db:
|
||||
rows = (await db.execute(text("""
|
||||
SELECT h.id AS hotel_id, h.name,
|
||||
hh.fresh_as_of, hh.stale_since
|
||||
FROM direct_competitor_hotels h
|
||||
JOIN direct_scrape_health hh ON hh.hotel_id = h.id
|
||||
WHERE hh.state = 'stale' AND h.scrape_enabled = true
|
||||
ORDER BY h.name
|
||||
"""))).mappings().all()
|
||||
return {"stale": [dict(r) for r in rows]}
|
||||
|
||||
|
||||
@router.post("/health/check")
|
||||
async def scrape_health_check(background_tasks: BackgroundTasks, user=Depends(get_current_user)):
|
||||
"""Re-evaluate scrape freshness on demand (also runs automatically after each scrape)."""
|
||||
require_cap(user, "manage_hotels")
|
||||
from jobs.check_scrape_health import check_scrape_health
|
||||
background_tasks.add_task(check_scrape_health)
|
||||
return {"status": "checking"}
|
||||
|
||||
|
||||
@router.post("/hotels", status_code=201)
|
||||
async def create_hotel(body: HotelCreate, user=Depends(get_current_user)):
|
||||
require_cap(user, "manage_hotels")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue