Wire up direct-rates column; remove dormant pause-on-block flow
Direct-rates sub-row in Market View was dead: the frontend filters
competitors on direct_hotel_id but /matrix and /hotels never returned it.
Add direct_hotel_id to both queries (+ HotelResponse), and only render the
"Direct" sub-row when a hotel actually has a non-null direct rate (was
rendering all-dashes on an empty {} object).
Remove the pause-on-block flow entirely — dormant since rotate-on-block
replaced it (nothing set booking_scraper_paused=true after set_scraper_paused
was dropped): is_scraper_paused, /config/unpause, the /scrape paused guard,
ScraperStatusResponse.paused/pause_until, and the frontend Paused badge +
Unpause button. Trim now-unused datetime import.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
1dc8c0a945
commit
74ae94671b
5 changed files with 15 additions and 103 deletions
|
|
@ -53,6 +53,7 @@ class HotelResponse(BaseModel):
|
|||
notes: Optional[str]
|
||||
first_seen_at: Optional[datetime]
|
||||
last_seen_at: Optional[datetime]
|
||||
direct_hotel_id: Optional[int] = None
|
||||
|
||||
|
||||
class RateResponse(BaseModel):
|
||||
|
|
@ -74,8 +75,6 @@ class RateResponse(BaseModel):
|
|||
|
||||
class ScraperStatusResponse(BaseModel):
|
||||
enabled: bool
|
||||
paused: bool
|
||||
pause_until: Optional[str]
|
||||
backend: str
|
||||
location_configured: bool
|
||||
location_name: Optional[str]
|
||||
|
|
@ -98,8 +97,6 @@ async def get_scraper_status(
|
|||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key IN (
|
||||
'booking_scraper_enabled',
|
||||
'booking_scraper_paused',
|
||||
'booking_scraper_pause_until',
|
||||
'booking_scraper_backend'
|
||||
)
|
||||
""")
|
||||
|
|
@ -138,8 +135,6 @@ async def get_scraper_status(
|
|||
|
||||
return ScraperStatusResponse(
|
||||
enabled=config.get('booking_scraper_enabled', 'false') == 'true',
|
||||
paused=config.get('booking_scraper_paused', 'false') == 'true',
|
||||
pause_until=config.get('booking_scraper_pause_until'),
|
||||
backend=config.get('booking_scraper_backend', 'playwright_local'),
|
||||
location_configured=location_row is not None,
|
||||
location_name=location_row.location_name if location_row else None,
|
||||
|
|
@ -349,19 +344,6 @@ async def enable_scraper(
|
|||
return {"status": "success", "enabled": enabled}
|
||||
|
||||
|
||||
@router.post("/config/unpause")
|
||||
async def unpause_scraper(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Manually unpause the scraper (clears blocking pause)."""
|
||||
await db.execute(
|
||||
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "success", "message": "Scraper unpaused"}
|
||||
|
||||
|
||||
# ============================================
|
||||
# MANUAL SCRAPE TRIGGER
|
||||
# ============================================
|
||||
|
|
@ -420,16 +402,12 @@ async def trigger_manual_scrape(
|
|||
if not location_result.fetchone():
|
||||
raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
|
||||
|
||||
# Check if paused
|
||||
paused_result = await db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
|
||||
)
|
||||
paused_row = paused_result.fetchone()
|
||||
if paused_row and paused_row.config_value == 'true':
|
||||
raise HTTPException(status_code=400, detail="Scraper is currently paused. Use /unpause first or wait for cooldown.")
|
||||
|
||||
# Only one scrape at a time — concurrent Chromium runs cause the page
|
||||
# timeouts that produce partial results
|
||||
#
|
||||
# Best-effort early 409: the background task re-acquires the lock and will
|
||||
# no-op (logging "another scrape is running") if it loses a millisecond-
|
||||
# window race, so no double-run can slip through here.
|
||||
from services.booking_scraper import SCRAPE_LOCK
|
||||
if SCRAPE_LOCK.locked():
|
||||
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
|
||||
|
|
@ -463,7 +441,8 @@ async def list_hotels(
|
|||
query = """
|
||||
SELECT id, booking_com_id, name, booking_com_url,
|
||||
star_rating, review_score, review_count,
|
||||
tier, display_order, notes, first_seen_at, last_seen_at
|
||||
tier, display_order, notes, first_seen_at, last_seen_at,
|
||||
direct_hotel_id
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE
|
||||
"""
|
||||
|
|
@ -492,7 +471,8 @@ async def list_hotels(
|
|||
display_order=row.display_order,
|
||||
notes=row.notes,
|
||||
first_seen_at=row.first_seen_at,
|
||||
last_seen_at=row.last_seen_at
|
||||
last_seen_at=row.last_seen_at,
|
||||
direct_hotel_id=row.direct_hotel_id
|
||||
)
|
||||
for row in result.fetchall()
|
||||
]
|
||||
|
|
@ -605,7 +585,8 @@ async def get_competitor_matrix(
|
|||
# Get hotels
|
||||
hotels_result = await db.execute(
|
||||
text(f"""
|
||||
SELECT id, name, tier, display_order, star_rating, review_score, booking_com_url
|
||||
SELECT id, name, tier, display_order, star_rating, review_score,
|
||||
booking_com_url, direct_hotel_id
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE AND {tier_filter.replace('h.', '')}
|
||||
ORDER BY display_order, name
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import asyncio
|
|||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import List, Optional, Dict, Any, Tuple
|
||||
|
||||
|
|
@ -84,36 +84,6 @@ def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
|
|||
}
|
||||
|
||||
|
||||
async def is_scraper_paused(db: Session) -> bool:
|
||||
"""Check if scraper is currently paused due to blocking."""
|
||||
result = db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
|
||||
).fetchone()
|
||||
|
||||
if not result or result.config_value != 'true':
|
||||
return False
|
||||
|
||||
# Check if pause period has expired
|
||||
pause_until_result = db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'")
|
||||
).fetchone()
|
||||
|
||||
if pause_until_result and pause_until_result.config_value:
|
||||
try:
|
||||
pause_until = datetime.fromisoformat(pause_until_result.config_value)
|
||||
if datetime.now() >= pause_until:
|
||||
# Pause expired, reset
|
||||
db.execute(
|
||||
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
|
||||
)
|
||||
db.commit()
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def save_hotel(db: Session, hotel: HotelData) -> int:
|
||||
"""
|
||||
Save or update a hotel in the database.
|
||||
|
|
@ -557,13 +527,6 @@ async def _run_manual_scrape_locked(
|
|||
from_date: date,
|
||||
to_date: date
|
||||
) -> Dict[str, Any]:
|
||||
# Check if paused
|
||||
if await is_scraper_paused(db):
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'Scraper is currently paused due to blocking. Try again later.',
|
||||
}
|
||||
|
||||
# Get config
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
|
|
@ -745,13 +708,6 @@ async def process_queue(db: Session) -> Dict[str, Any]:
|
|||
|
||||
|
||||
async def _process_queue_locked(db: Session) -> Dict[str, Any]:
|
||||
# Check if paused
|
||||
if await is_scraper_paused(db):
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'Scraper is currently paused due to blocking.',
|
||||
}
|
||||
|
||||
# Get config
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue