Wire up direct-rates column; remove dormant pause-on-block flow
Direct-rates sub-row in Market View was dead: the frontend filters
competitors on direct_hotel_id but /matrix and /hotels never returned it.
Add direct_hotel_id to both queries (+ HotelResponse), and only render the
"Direct" sub-row when a hotel actually has a non-null direct rate (was
rendering all-dashes on an empty {} object).
Remove the pause-on-block flow entirely — dormant since rotate-on-block
replaced it (nothing set booking_scraper_paused=true after set_scraper_paused
was dropped): is_scraper_paused, /config/unpause, the /scrape paused guard,
ScraperStatusResponse.paused/pause_until, and the frontend Paused badge +
Unpause button. Trim now-unused datetime import.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
1dc8c0a945
commit
74ae94671b
5 changed files with 15 additions and 103 deletions
|
|
@ -53,6 +53,7 @@ class HotelResponse(BaseModel):
|
|||
notes: Optional[str]
|
||||
first_seen_at: Optional[datetime]
|
||||
last_seen_at: Optional[datetime]
|
||||
direct_hotel_id: Optional[int] = None
|
||||
|
||||
|
||||
class RateResponse(BaseModel):
|
||||
|
|
@ -74,8 +75,6 @@ class RateResponse(BaseModel):
|
|||
|
||||
class ScraperStatusResponse(BaseModel):
|
||||
enabled: bool
|
||||
paused: bool
|
||||
pause_until: Optional[str]
|
||||
backend: str
|
||||
location_configured: bool
|
||||
location_name: Optional[str]
|
||||
|
|
@ -98,8 +97,6 @@ async def get_scraper_status(
|
|||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key IN (
|
||||
'booking_scraper_enabled',
|
||||
'booking_scraper_paused',
|
||||
'booking_scraper_pause_until',
|
||||
'booking_scraper_backend'
|
||||
)
|
||||
""")
|
||||
|
|
@ -138,8 +135,6 @@ async def get_scraper_status(
|
|||
|
||||
return ScraperStatusResponse(
|
||||
enabled=config.get('booking_scraper_enabled', 'false') == 'true',
|
||||
paused=config.get('booking_scraper_paused', 'false') == 'true',
|
||||
pause_until=config.get('booking_scraper_pause_until'),
|
||||
backend=config.get('booking_scraper_backend', 'playwright_local'),
|
||||
location_configured=location_row is not None,
|
||||
location_name=location_row.location_name if location_row else None,
|
||||
|
|
@ -349,19 +344,6 @@ async def enable_scraper(
|
|||
return {"status": "success", "enabled": enabled}
|
||||
|
||||
|
||||
@router.post("/config/unpause")
|
||||
async def unpause_scraper(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Manually unpause the scraper (clears blocking pause)."""
|
||||
await db.execute(
|
||||
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "success", "message": "Scraper unpaused"}
|
||||
|
||||
|
||||
# ============================================
|
||||
# MANUAL SCRAPE TRIGGER
|
||||
# ============================================
|
||||
|
|
@ -420,16 +402,12 @@ async def trigger_manual_scrape(
|
|||
if not location_result.fetchone():
|
||||
raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
|
||||
|
||||
# Check if paused
|
||||
paused_result = await db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
|
||||
)
|
||||
paused_row = paused_result.fetchone()
|
||||
if paused_row and paused_row.config_value == 'true':
|
||||
raise HTTPException(status_code=400, detail="Scraper is currently paused. Use /unpause first or wait for cooldown.")
|
||||
|
||||
# Only one scrape at a time — concurrent Chromium runs cause the page
|
||||
# timeouts that produce partial results
|
||||
#
|
||||
# Best-effort early 409: the background task re-acquires the lock and will
|
||||
# no-op (logging "another scrape is running") if it loses a millisecond-
|
||||
# window race, so no double-run can slip through here.
|
||||
from services.booking_scraper import SCRAPE_LOCK
|
||||
if SCRAPE_LOCK.locked():
|
||||
raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
|
||||
|
|
@ -463,7 +441,8 @@ async def list_hotels(
|
|||
query = """
|
||||
SELECT id, booking_com_id, name, booking_com_url,
|
||||
star_rating, review_score, review_count,
|
||||
tier, display_order, notes, first_seen_at, last_seen_at
|
||||
tier, display_order, notes, first_seen_at, last_seen_at,
|
||||
direct_hotel_id
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE
|
||||
"""
|
||||
|
|
@ -492,7 +471,8 @@ async def list_hotels(
|
|||
display_order=row.display_order,
|
||||
notes=row.notes,
|
||||
first_seen_at=row.first_seen_at,
|
||||
last_seen_at=row.last_seen_at
|
||||
last_seen_at=row.last_seen_at,
|
||||
direct_hotel_id=row.direct_hotel_id
|
||||
)
|
||||
for row in result.fetchall()
|
||||
]
|
||||
|
|
@ -605,7 +585,8 @@ async def get_competitor_matrix(
|
|||
# Get hotels
|
||||
hotels_result = await db.execute(
|
||||
text(f"""
|
||||
SELECT id, name, tier, display_order, star_rating, review_score, booking_com_url
|
||||
SELECT id, name, tier, display_order, star_rating, review_score,
|
||||
booking_com_url, direct_hotel_id
|
||||
FROM booking_com_hotels
|
||||
WHERE is_active = TRUE AND {tier_filter.replace('h.', '')}
|
||||
ORDER BY display_order, name
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import asyncio
|
|||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import List, Optional, Dict, Any, Tuple
|
||||
|
||||
|
|
@ -84,36 +84,6 @@ def get_scrape_config(db: Session) -> Optional[Dict[str, Any]]:
|
|||
}
|
||||
|
||||
|
||||
async def is_scraper_paused(db: Session) -> bool:
|
||||
"""Check if scraper is currently paused due to blocking."""
|
||||
result = db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_paused'")
|
||||
).fetchone()
|
||||
|
||||
if not result or result.config_value != 'true':
|
||||
return False
|
||||
|
||||
# Check if pause period has expired
|
||||
pause_until_result = db.execute(
|
||||
text("SELECT config_value FROM system_config WHERE config_key = 'booking_scraper_pause_until'")
|
||||
).fetchone()
|
||||
|
||||
if pause_until_result and pause_until_result.config_value:
|
||||
try:
|
||||
pause_until = datetime.fromisoformat(pause_until_result.config_value)
|
||||
if datetime.now() >= pause_until:
|
||||
# Pause expired, reset
|
||||
db.execute(
|
||||
text("UPDATE system_config SET config_value = 'false' WHERE config_key = 'booking_scraper_paused'")
|
||||
)
|
||||
db.commit()
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def save_hotel(db: Session, hotel: HotelData) -> int:
|
||||
"""
|
||||
Save or update a hotel in the database.
|
||||
|
|
@ -557,13 +527,6 @@ async def _run_manual_scrape_locked(
|
|||
from_date: date,
|
||||
to_date: date
|
||||
) -> Dict[str, Any]:
|
||||
# Check if paused
|
||||
if await is_scraper_paused(db):
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'Scraper is currently paused due to blocking. Try again later.',
|
||||
}
|
||||
|
||||
# Get config
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
|
|
@ -745,13 +708,6 @@ async def process_queue(db: Session) -> Dict[str, Any]:
|
|||
|
||||
|
||||
async def _process_queue_locked(db: Session) -> Dict[str, Any]:
|
||||
# Check if paused
|
||||
if await is_scraper_paused(db):
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'Scraper is currently paused due to blocking.',
|
||||
}
|
||||
|
||||
# Get config
|
||||
config = get_scrape_config(db)
|
||||
if not config:
|
||||
|
|
|
|||
|
|
@ -32,8 +32,6 @@ interface ScrapeJob {
|
|||
|
||||
interface ScraperStatus {
|
||||
enabled: boolean
|
||||
paused: boolean
|
||||
pause_until: string | null
|
||||
backend: string
|
||||
location_configured: boolean
|
||||
location_name: string | null
|
||||
|
|
@ -274,14 +272,6 @@ const StatusPanel: React.FC<{ status: ScraperStatus | undefined, isLoading: bool
|
|||
{status.enabled ? 'Enabled' : 'Disabled'}
|
||||
</span>
|
||||
</div>
|
||||
{status.paused && (
|
||||
<div style={styles.statusItem}>
|
||||
<span style={styles.statusLabel}>Status</span>
|
||||
<span style={badgeStyle('warning')}>
|
||||
Paused{status.pause_until ? ` until ${formatDateTime(status.pause_until)}` : ''}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div style={styles.statusItem}>
|
||||
<span style={styles.statusLabel}>Location</span>
|
||||
<span style={{ fontSize: '13px', color: status.location_configured ? 'var(--text-dark)' : 'var(--text-mid)' }}>
|
||||
|
|
@ -379,11 +369,6 @@ const SettingsTab: React.FC = () => {
|
|||
onSuccess: () => queryClient.invalidateQueries({ queryKey: ['scraper-status'] }),
|
||||
})
|
||||
|
||||
const unpauseMutation = useMutation({
|
||||
mutationFn: async () => (await api.post('/competitors/config/unpause')).data,
|
||||
onSuccess: () => queryClient.invalidateQueries({ queryKey: ['scraper-status'] }),
|
||||
})
|
||||
|
||||
const scrapeMutation = useMutation({
|
||||
mutationFn: async () => {
|
||||
return (await api.post('/competitors/scrape', {
|
||||
|
|
@ -486,14 +471,6 @@ const SettingsTab: React.FC = () => {
|
|||
>
|
||||
{status?.enabled ? 'Disable Scraper' : 'Enable Scraper'}
|
||||
</button>
|
||||
{status?.paused && (
|
||||
<button
|
||||
onClick={() => unpauseMutation.mutate()}
|
||||
style={buttonStyle('primary')}
|
||||
>
|
||||
Unpause Scraper
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -1419,8 +1396,9 @@ const RateMatrixTab: React.FC = () => {
|
|||
)
|
||||
})}
|
||||
</tr>
|
||||
{/* Direct rates sub-row */}
|
||||
{showDirect && hotel.tier === 'competitor' && (directRatesMap?.[hotel.id] != null) && (
|
||||
{/* Direct rates sub-row — only when this hotel actually has direct rates */}
|
||||
{showDirect && hotel.tier === 'competitor' &&
|
||||
Object.values(directRatesMap?.[hotel.id] || {}).some(v => v != null) && (
|
||||
<tr style={{ background: '#fafbfc' }}>
|
||||
<td style={{ ...styles.matrixTd, ...styles.stickyCol, paddingLeft: 28, fontSize: 11, color: 'var(--text-mid)', fontStyle: 'italic' }}>
|
||||
Direct
|
||||
|
|
|
|||
|
|
@ -536,7 +536,6 @@ function SystemTab({ config, isLoading }: { config: SystemConfig | undefined; is
|
|||
|
||||
const displayKeys = [
|
||||
'booking_scraper_enabled',
|
||||
'booking_scraper_paused',
|
||||
'booking_scraper_backend',
|
||||
'booking_scraper_daily_time',
|
||||
'booking_proxy_enabled',
|
||||
|
|
|
|||
|
|
@ -38,8 +38,6 @@ export interface RateMatrixEntry {
|
|||
|
||||
export interface ScraperStatus {
|
||||
enabled: boolean
|
||||
paused: boolean
|
||||
pause_until: string | null
|
||||
backend: string
|
||||
daily_time: string
|
||||
last_batch: {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue