Auto-recheck parity after scrape + manual Recheck button + faster badge
Parity logic: - Extract _fetch_latest_alerts() and _upsert_alerts() helpers so the alert upsert loop is no longer duplicated between the daily job and per-date runs - Add run_parity_check_for_date(date) which runs the full comparison + alert upsert for a single date Scraper integration: - _safe_parity_recheck(date) wrapper (never raises) called after each successful date in both search-results and hotel-page workers; hotel-page mode waits until all hotels for the date are done before rechecking API: - POST /competitors/parity/check-date?rate_date=YYYY-MM-DD for manual recheck Frontend: - Recheck button on every parity alert row (all statuses); invalidates alerts list and badge count on success - parity-alert-count badge polls every 15s (was 60s) so new alerts from the scheduled job or post-scrape rechecks appear quickly Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
8a80c66a3b
commit
cab1a39503
5 changed files with 146 additions and 66 deletions
|
|
@ -4,7 +4,7 @@ Booking.com rate scraping, hotel management, and competitor comparison
|
||||||
"""
|
"""
|
||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List, Dict, Any
|
||||||
from datetime import date, datetime, timedelta
|
from datetime import date, datetime, timedelta
|
||||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks, Query
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
@ -1042,6 +1042,18 @@ async def trigger_parity_check(
|
||||||
return await loop.run_in_executor(None, run_parity_check)
|
return await loop.run_in_executor(None, run_parity_check)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/parity/check-date")
|
||||||
|
async def trigger_parity_check_for_date(
|
||||||
|
rate_date: date = Query(..., description="Date to recheck (YYYY-MM-DD)"),
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
"""Recheck parity for a single date using the latest scraped rates."""
|
||||||
|
import asyncio
|
||||||
|
from jobs.check_rate_parity import run_parity_check_for_date
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
return await loop.run_in_executor(None, run_parity_check_for_date, rate_date)
|
||||||
|
|
||||||
|
|
||||||
# ============================================
|
# ============================================
|
||||||
# PARITY ALERTS
|
# PARITY ALERTS
|
||||||
# ============================================
|
# ============================================
|
||||||
|
|
|
||||||
|
|
@ -244,42 +244,24 @@ def gather_comparisons(db, start: date, end: date, cfg: dict) -> list:
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def run_parity_check() -> dict:
|
def _fetch_latest_alerts(db, start: date, end: date) -> dict:
|
||||||
db = SyncSessionLocal()
|
"""Return {rate_date: {id, status, diff}} for the most-recent alert per date in [start, end]."""
|
||||||
try:
|
rows = db.execute(text("""
|
||||||
cfg = get_parity_config(db)
|
|
||||||
if not cfg["enabled"]:
|
|
||||||
logger.info("Parity check skipped (disabled)")
|
|
||||||
return {"status": "disabled"}
|
|
||||||
|
|
||||||
today = date.today()
|
|
||||||
start = today
|
|
||||||
end = today + timedelta(days=HORIZON_DAYS)
|
|
||||||
|
|
||||||
# Resolve any lingering active alerts for dates that have already passed
|
|
||||||
db.execute(text("""
|
|
||||||
UPDATE rate_parity_alerts
|
|
||||||
SET alert_status = 'resolved'
|
|
||||||
WHERE alert_status = 'active'
|
|
||||||
AND rate_date < :today
|
|
||||||
"""), {"today": today})
|
|
||||||
|
|
||||||
comparisons = gather_comparisons(db, start, end, cfg)
|
|
||||||
|
|
||||||
# Latest alert per date in the horizon
|
|
||||||
alert_rows = db.execute(text("""
|
|
||||||
SELECT DISTINCT ON (rate_date)
|
SELECT DISTINCT ON (rate_date)
|
||||||
id, rate_date, alert_status, difference_pct
|
id, rate_date, alert_status, difference_pct
|
||||||
FROM rate_parity_alerts
|
FROM rate_parity_alerts
|
||||||
WHERE rate_date BETWEEN :fd AND :td
|
WHERE rate_date BETWEEN :fd AND :td
|
||||||
ORDER BY rate_date, created_at DESC
|
ORDER BY rate_date, created_at DESC
|
||||||
"""), {"fd": start, "td": end}).fetchall()
|
"""), {"fd": start, "td": end}).fetchall()
|
||||||
latest_alert = {r[1]: {"id": r[0], "status": r[2], "diff": float(r[3] or 0)} for r in alert_rows}
|
return {r[1]: {"id": r[0], "status": r[2], "diff": float(r[3] or 0)} for r in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def _upsert_alerts(db, comparisons: list, latest_alert: dict) -> tuple:
|
||||||
|
"""Apply parity comparison results to the alerts table.
|
||||||
|
Returns (created, updated, resolved)."""
|
||||||
created = updated = resolved = 0
|
created = updated = resolved = 0
|
||||||
for c in comparisons:
|
for c in comparisons:
|
||||||
alert = latest_alert.get(c["date"])
|
alert = latest_alert.get(c["date"])
|
||||||
# e.g. "DIRECT B&B PPAY vs BC prepaid/B&B"
|
|
||||||
category_label = f"{c['newbook_tariff']} vs BC {c['booking_basis']}"[:255]
|
category_label = f"{c['newbook_tariff']} vs BC {c['booking_basis']}"[:255]
|
||||||
|
|
||||||
if c["breach"]:
|
if c["breach"]:
|
||||||
|
|
@ -321,6 +303,58 @@ def run_parity_check() -> dict:
|
||||||
)
|
)
|
||||||
resolved += 1
|
resolved += 1
|
||||||
|
|
||||||
|
return created, updated, resolved
|
||||||
|
|
||||||
|
|
||||||
|
def run_parity_check_for_date(rate_date: date) -> dict:
|
||||||
|
"""Run the parity check for a single date.
|
||||||
|
Called after a date is re-scraped, or triggered manually from the UI."""
|
||||||
|
db = SyncSessionLocal()
|
||||||
|
try:
|
||||||
|
cfg = get_parity_config(db)
|
||||||
|
if not cfg["enabled"]:
|
||||||
|
return {"status": "disabled"}
|
||||||
|
|
||||||
|
comparisons = gather_comparisons(db, rate_date, rate_date, cfg)
|
||||||
|
if not comparisons:
|
||||||
|
return {"status": "ok", "date": str(rate_date), "no_data": True}
|
||||||
|
|
||||||
|
latest_alert = _fetch_latest_alerts(db, rate_date, rate_date)
|
||||||
|
created, updated, resolved = _upsert_alerts(db, comparisons, latest_alert)
|
||||||
|
db.commit()
|
||||||
|
return {"status": "ok", "date": str(rate_date),
|
||||||
|
"created": created, "updated": updated, "resolved": resolved}
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def run_parity_check() -> dict:
|
||||||
|
db = SyncSessionLocal()
|
||||||
|
try:
|
||||||
|
cfg = get_parity_config(db)
|
||||||
|
if not cfg["enabled"]:
|
||||||
|
logger.info("Parity check skipped (disabled)")
|
||||||
|
return {"status": "disabled"}
|
||||||
|
|
||||||
|
today = date.today()
|
||||||
|
start = today
|
||||||
|
end = today + timedelta(days=HORIZON_DAYS)
|
||||||
|
|
||||||
|
# Resolve any lingering active alerts for dates that have already passed
|
||||||
|
db.execute(text("""
|
||||||
|
UPDATE rate_parity_alerts
|
||||||
|
SET alert_status = 'resolved'
|
||||||
|
WHERE alert_status = 'active'
|
||||||
|
AND rate_date < :today
|
||||||
|
"""), {"today": today})
|
||||||
|
|
||||||
|
comparisons = gather_comparisons(db, start, end, cfg)
|
||||||
|
latest_alert = _fetch_latest_alerts(db, start, end)
|
||||||
|
created, updated, resolved = _upsert_alerts(db, comparisons, latest_alert)
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
summary = {
|
summary = {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
|
|
|
||||||
|
|
@ -613,6 +613,7 @@ async def _scrape_hotels_concurrent(
|
||||||
backend = PlaywrightHotelPageBackend(proxy_config=proxy_util.load_config(wdb))
|
backend = PlaywrightHotelPageBackend(proxy_config=proxy_util.load_config(wdb))
|
||||||
try:
|
try:
|
||||||
for rate_date in date_shard:
|
for rate_date in date_shard:
|
||||||
|
date_rates = 0
|
||||||
for hotel in hotels_seen:
|
for hotel in hotels_seen:
|
||||||
try:
|
try:
|
||||||
result = await scrape_hotel_date(wdb, hotel, rate_date, backend, batch_id, adults)
|
result = await scrape_hotel_date(wdb, hotel, rate_date, backend, batch_id, adults)
|
||||||
|
|
@ -632,12 +633,16 @@ async def _scrape_hotels_concurrent(
|
||||||
elif result['success']:
|
elif result['success']:
|
||||||
acc['rates'] += result['rates_count']
|
acc['rates'] += result['rates_count']
|
||||||
acc['completed'] += 1
|
acc['completed'] += 1
|
||||||
|
date_rates += result['rates_count']
|
||||||
_increment_batch_progress(wdb, batch_id,
|
_increment_batch_progress(wdb, batch_id,
|
||||||
delta_rates=result['rates_count'],
|
delta_rates=result['rates_count'],
|
||||||
delta_completed=1)
|
delta_completed=1)
|
||||||
else:
|
else:
|
||||||
acc['failed'] += 1
|
acc['failed'] += 1
|
||||||
_increment_batch_progress(wdb, batch_id, delta_failed=1)
|
_increment_batch_progress(wdb, batch_id, delta_failed=1)
|
||||||
|
|
||||||
|
if date_rates > 0:
|
||||||
|
_safe_parity_recheck(rate_date)
|
||||||
finally:
|
finally:
|
||||||
try:
|
try:
|
||||||
await backend.close()
|
await backend.close()
|
||||||
|
|
@ -687,6 +692,15 @@ def _increment_batch_progress(
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_parity_recheck(rate_date: date):
|
||||||
|
"""Re-run parity check for one date after it has been successfully scraped. Never raises."""
|
||||||
|
try:
|
||||||
|
from jobs.check_rate_parity import run_parity_check_for_date
|
||||||
|
run_parity_check_for_date(rate_date)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Post-scrape parity recheck for {rate_date} failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
|
def _safe_mark_queue(db: Session, queue_id: Optional[int], status: str, error: str = None):
|
||||||
"""mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
|
"""mark_queue_item that never raises — a marking failure shouldn't kill a worker."""
|
||||||
if queue_id is None:
|
if queue_id is None:
|
||||||
|
|
@ -749,6 +763,7 @@ async def _scrape_dates_concurrent(
|
||||||
delta_hotels=result['hotels_count'],
|
delta_hotels=result['hotels_count'],
|
||||||
delta_rates=result['rates_count'],
|
delta_rates=result['rates_count'],
|
||||||
delta_completed=1)
|
delta_completed=1)
|
||||||
|
_safe_parity_recheck(rate_date)
|
||||||
else:
|
else:
|
||||||
acc['failed'] += 1
|
acc['failed'] += 1
|
||||||
_safe_mark_queue(wdb, queue_id, 'failed', result.get('error'))
|
_safe_mark_queue(wdb, queue_id, 'failed', result.get('error'))
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ export default function Layout({ children }: { children: ReactNode }) {
|
||||||
const { data: alertCount } = useQuery<number>({
|
const { data: alertCount } = useQuery<number>({
|
||||||
queryKey: ['parity-alert-count'],
|
queryKey: ['parity-alert-count'],
|
||||||
queryFn: () => api.get('/competitors/parity/alerts?status=active').then(r => r.data.length),
|
queryFn: () => api.get('/competitors/parity/alerts?status=active').then(r => r.data.length),
|
||||||
refetchInterval: 60_000,
|
refetchInterval: 15_000,
|
||||||
enabled: can(user, 'view_competitors'),
|
enabled: can(user, 'view_competitors'),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -843,6 +843,15 @@ const ParityAlertsTab: React.FC = () => {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const recheckMutation = useMutation({
|
||||||
|
mutationFn: async (rateDate: string) =>
|
||||||
|
(await api.post(`/competitors/parity/check-date?rate_date=${rateDate}`)).data,
|
||||||
|
onSuccess: () => {
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['parity-alerts'] })
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['parity-alert-count'] })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
const markupUnit = parityConfig?.['parity_markup_unit'] ?? 'pct'
|
const markupUnit = parityConfig?.['parity_markup_unit'] ?? 'pct'
|
||||||
const toleranceUnit = parityConfig?.['parity_tolerance_unit'] ?? 'pct'
|
const toleranceUnit = parityConfig?.['parity_tolerance_unit'] ?? 'pct'
|
||||||
const markupVal = parityConfig?.['parity_markup_value'] ?? parityConfig?.['parity_expected_markup_pct'] ?? '0'
|
const markupVal = parityConfig?.['parity_markup_value'] ?? parityConfig?.['parity_expected_markup_pct'] ?? '0'
|
||||||
|
|
@ -945,19 +954,29 @@ const ParityAlertsTab: React.FC = () => {
|
||||||
{a.alert_status}
|
{a.alert_status}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td style={tdStyle}>
|
<td style={mergeStyles(tdStyle, { whiteSpace: 'nowrap' })}>
|
||||||
|
<div style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||||
{a.alert_status === 'active' && (
|
{a.alert_status === 'active' && (
|
||||||
<button
|
<button
|
||||||
style={buttonStyle('outline', 'small')}
|
style={buttonStyle('outline', 'small')}
|
||||||
disabled={ackMutation.isPending}
|
disabled={ackMutation.isPending || recheckMutation.isPending}
|
||||||
onClick={() => ackMutation.mutate(a.id)}
|
onClick={() => ackMutation.mutate(a.id)}
|
||||||
>
|
>
|
||||||
Acknowledge
|
Acknowledge
|
||||||
</button>
|
</button>
|
||||||
)}
|
)}
|
||||||
|
<button
|
||||||
|
style={buttonStyle('outline', 'small')}
|
||||||
|
disabled={recheckMutation.isPending || ackMutation.isPending}
|
||||||
|
onClick={() => recheckMutation.mutate(a.rate_date)}
|
||||||
|
title="Re-run parity check using the latest scraped rates for this date"
|
||||||
|
>
|
||||||
|
Recheck
|
||||||
|
</button>
|
||||||
{a.alert_status === 'acknowledged' && a.acknowledged_by && (
|
{a.alert_status === 'acknowledged' && a.acknowledged_by && (
|
||||||
<span style={{ fontSize: 11, color: 'var(--text-mid)' }}>by {a.acknowledged_by}</span>
|
<span style={{ fontSize: 11, color: 'var(--text-mid)' }}>by {a.acknowledged_by}</span>
|
||||||
)}
|
)}
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
))}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue