Add hotel discovery scrape, manual hotel add, and competitor-only filtering

- Scraper: restrict hotel-page scraper to 'competitor' tier only (was own+competitor); own hotel rates come from Newbook API, not Booking.com
- Backend: POST /competitors/discover — runs search-results scrape for one date to find market hotels regardless of current backend setting
- Backend: POST /competitors/hotels — add hotel manually from Booking.com URL + name + tier; upserts on slug conflict
- Frontend (Settings tab): Discover Market Hotels button added below manual date-range scrape
- Frontend (Hotels tab): Add Hotel form at top — paste URL (auto-derives name from slug), choose tier, submit

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/api/competitors.py

@@ -452,10 +452,113 @@ async def reset_scraper(
     return result
 
 
+@router.post("/discover")
+async def trigger_discovery_scrape(
+    background_tasks: BackgroundTasks,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user),
+):
+    """
+    Run a search-results scrape for one date to discover market hotels.
+    Always uses playwright_local (search-results) regardless of the configured
+    backend setting — intended for discovery, not rate tracking.
+    """
+    if not (current_user.get('is_admin') or 'manage_scraper' in (current_user.get('caps') or [])):
+        raise HTTPException(status_code=403, detail="manage_scraper capability required")
+
+    location_result = await db.execute(
+        text("SELECT id FROM booking_scrape_config WHERE is_active = TRUE LIMIT 1")
+    )
+    if not location_result.fetchone():
+        raise HTTPException(status_code=400, detail="No scrape location configured. Set location first.")
+
+    from services.booking_scraper import get_lock_status
+    if get_lock_status()["locked"] or _SCRAPE_PENDING.is_set():
+        raise HTTPException(status_code=409, detail="A scrape is already running. Try again when it finishes.")
+
+    def _run():
+        from services.booking_scraper import run_discovery_scrape
+        import asyncio
+        sync_db = SyncSessionLocal()
+        try:
+            asyncio.run(run_discovery_scrape(sync_db))
+        finally:
+            sync_db.close()
+            _SCRAPE_PENDING.clear()
+
+    _SCRAPE_PENDING.set()
+    background_tasks.add_task(_run)
+    return {"status": "started", "message": "Discovery scrape started. Check /status for progress."}
+
+
 # ============================================
 # HOTELS MANAGEMENT
 # ============================================
 
+
+class HotelManualCreate(BaseModel):
+    booking_com_url: str
+    name: str
+    tier: str = 'market'  # 'own', 'competitor', 'market'
+
+
+@router.post("/hotels", response_model=HotelResponse)
+async def create_hotel_manually(
+    payload: HotelManualCreate,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(get_current_user),
+):
+    """Add a hotel manually by pasting its Booking.com URL."""
+    if not (current_user.get('is_admin') or 'manage_hotels' in (current_user.get('caps') or [])):
+        raise HTTPException(status_code=403, detail="manage_hotels capability required")
+
+    if payload.tier not in ('own', 'competitor', 'market'):
+        raise HTTPException(status_code=400, detail="tier must be own, competitor, or market")
+
+    # Extract slug from URL as booking_com_id
+    import re
+    m = re.search(r'/hotel/\w+/([^.?#]+)', payload.booking_com_url)
+    slug = m.group(1) if m else None
+    if not slug:
+        raise HTTPException(status_code=400, detail="Could not parse hotel slug from URL. Expected a URL like booking.com/hotel/gb/hotel-name.en-gb.html")
+
+    # Upsert — if slug already exists, update tier/url/name
+    result = await db.execute(
+        text("""
+            INSERT INTO booking_com_hotels
+                (booking_com_id, name, booking_com_url, tier, is_active, first_seen_at, last_seen_at)
+            VALUES (:slug, :name, :url, :tier, TRUE, NOW(), NOW())
+            ON CONFLICT (booking_com_id) DO UPDATE SET
+                name          = EXCLUDED.name,
+                booking_com_url = EXCLUDED.booking_com_url,
+                tier          = EXCLUDED.tier,
+                is_active     = TRUE,
+                last_seen_at  = NOW()
+            RETURNING id, booking_com_id, name, booking_com_url, star_rating, review_score,
+                      review_count, tier, display_order, notes, first_seen_at, last_seen_at,
+                      direct_hotel_id
+        """),
+        {'slug': slug, 'name': payload.name.strip(), 'url': payload.booking_com_url.strip(), 'tier': payload.tier}
+    )
+    await db.commit()
+    row = result.fetchone()
+    return HotelResponse(
+        id=row.id,
+        booking_com_id=row.booking_com_id or '',
+        name=row.name,
+        booking_com_url=row.booking_com_url,
+        star_rating=float(row.star_rating) if row.star_rating else None,
+        review_score=float(row.review_score) if row.review_score else None,
+        review_count=row.review_count,
+        tier=row.tier,
+        display_order=row.display_order,
+        notes=row.notes,
+        first_seen_at=row.first_seen_at,
+        last_seen_at=row.last_seen_at,
+        direct_hotel_id=row.direct_hotel_id,
+    )
+
+
 @router.get("/hotels", response_model=List[HotelResponse])
 async def list_hotels(
     tier: Optional[str] = None,

backend/services/booking_scraper.py

@@ -295,16 +295,16 @@ def cleanup_stale_batches(db: Session, max_age_minutes: int = 60):
 
 
 def get_active_hotels(db: Session) -> List[Dict[str, Any]]:
-    """Return own + competitor hotels with a booking_com_url (for hotel-page scraping).
+    """Return competitor hotels with a booking_com_url (for hotel-page scraping).
-    Market-tier hotels are excluded — they were auto-discovered from search results and
+    Own and market hotels are excluded — market were auto-discovered from search results;
-    are not hotels we specifically want to track at rate-plan level."""
+    own hotel rates come from the Newbook API, not Booking.com scraping."""
     rows = db.execute(
         text("""
             SELECT id, booking_com_id, name, booking_com_url
             FROM booking_com_hotels
             WHERE is_active = TRUE
               AND booking_com_url IS NOT NULL
-              AND tier IN ('own', 'competitor')
+              AND tier = 'competitor'
             ORDER BY display_order, id
         """)
     ).fetchall()
@@ -884,6 +884,49 @@ async def _run_manual_scrape_locked(
         }
 
 
+async def run_discovery_scrape(db: Session) -> Dict[str, Any]:
+    """
+    Run a search-results scrape for a single date to discover market hotels.
+    Always uses playwright_local (search-results) regardless of the configured
+    backend — intended for hotel discovery, not rate tracking.
+    Returns new_hotels: count of hotels added to the DB this run.
+    """
+    config = get_scrape_config(db)
+    if not config:
+        return {'success': False, 'error': 'No scrape location configured.'}
+
+    hotels_before = db.execute(
+        text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE")
+    ).scalar() or 0
+
+    batch_id = create_scrape_batch(db, 'discovery')
+    check_in = date.today() + timedelta(days=14)
+    jobs: List[Tuple[date, Optional[int]]] = [(check_in, None)]
+
+    try:
+        agg = await _scrape_dates_concurrent(jobs, config, concurrency=1, batch_id=batch_id)
+        update_scrape_batch(
+            db, batch_id,
+            status='completed' if agg['completed'] else 'failed',
+            hotels_found=agg['hotels'],
+            rates_scraped=agg['rates'],
+        )
+
+        hotels_after = db.execute(
+            text("SELECT COUNT(*) FROM booking_com_hotels WHERE is_active = TRUE")
+        ).scalar() or 0
+
+        return {
+            'success': agg['completed'] > 0,
+            'hotels_found': agg['hotels'],
+            'new_hotels': max(0, hotels_after - hotels_before),
+        }
+    except Exception as e:
+        logger.error(f"Discovery scrape error: {e}")
+        update_scrape_batch(db, batch_id, status='failed', error_message=str(e))
+        return {'success': False, 'error': str(e)}
+
+
 # ============================================
 # QUEUE MANAGEMENT
 # ============================================

frontend/src/pages/MarketView.tsx

@@ -534,7 +534,7 @@ const SettingsTab: React.FC = () => {
       <div style={styles.card}>
         <h3 style={styles.cardTitle}>Manual Scrape</h3>
         <p style={styles.cardDescription}>
-          Trigger a one-off scrape for a date range. Runs in background.
+          Trigger a one-off competitor rate scrape for a date range. Runs in background.
         </p>
         <div style={styles.formRow}>
           <div style={styles.formGroup}>
@@ -579,6 +579,15 @@ const SettingsTab: React.FC = () => {
             {(scrapeMutation.error as any)?.response?.data?.detail || 'Failed to start scrape'}
           </p>
         )}
+
+        <hr style={{ margin: '20px 0', border: 'none', borderTop: '1px solid var(--border)' }} />
+
+        <h4 style={{ margin: '0 0 6px', fontSize: '14px', color: 'var(--text-dark)' }}>Discover Market Hotels</h4>
+        <p style={styles.cardDescription}>
+          Runs the search-results scraper for one date to find hotels in your configured location.
+          Newly found hotels are added as <em>Market</em> tier so you can review and reclassify them.
+        </p>
+        <DiscoverButton locationConfigured={!!status?.location_configured} />
       </div>
 
       {/* Scrape History */}
@@ -905,6 +914,131 @@ const ParityAlertsTab: React.FC = () => {
   )
 }
 
+const DiscoverButton: React.FC<{ locationConfigured: boolean }> = ({ locationConfigured }) => {
+  const qc = useQueryClient()
+  const m = useMutation({
+    mutationFn: () => api.post('/competitors/discover').then(r => r.data),
+    onSuccess: () => {
+      qc.invalidateQueries({ queryKey: ['scraper-status'] })
+      qc.invalidateQueries({ queryKey: ['scrape-history'] })
+    },
+  })
+  return (
+    <div>
+      <button
+        className="btn btn-outline"
+        onClick={() => m.mutate()}
+        disabled={m.isPending || !locationConfigured}
+        style={{ opacity: locationConfigured ? 1 : 0.5 }}
+      >
+        {m.isPending ? 'Discovering…' : 'Discover Market Hotels'}
+      </button>
+      {!locationConfigured && <p style={styles.hintText}>Configure a location first</p>}
+      {m.isSuccess && (
+        <p style={{ color: 'var(--success)', fontSize: '13px', marginTop: '8px' }}>
+          Discovery started — new hotels will appear in the Hotels tab as Market tier.
+        </p>
+      )}
+      {m.isError && (
+        <p style={{ color: 'var(--danger)', fontSize: '13px', marginTop: '8px' }}>
+          {(m.error as any)?.response?.data?.detail || 'Discovery failed'}
+        </p>
+      )}
+    </div>
+  )
+}
+
+
+const AddHotelForm: React.FC = () => {
+  const qc = useQueryClient()
+  const [url, setUrl] = useState('')
+  const [name, setName] = useState('')
+  const [tier, setTier] = useState('competitor')
+
+  // Auto-derive a display name from the URL slug when the user pastes a URL
+  const handleUrlChange = (raw: string) => {
+    setUrl(raw)
+    const m = raw.match(/\/hotel\/\w+\/([^.?#]+)/)
+    if (m) {
+      const derived = m[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase())
+      if (!name) setName(derived)
+    }
+  }
+
+  const m = useMutation({
+    mutationFn: () => api.post('/competitors/hotels', { booking_com_url: url.trim(), name: name.trim(), tier }).then(r => r.data),
+    onSuccess: () => {
+      qc.invalidateQueries({ queryKey: ['competitor-hotels'] })
+      setUrl('')
+      setName('')
+      setTier('competitor')
+    },
+  })
+
+  return (
+    <div className="card" style={{ marginBottom: 16 }}>
+      <div className="card-header">Add Hotel</div>
+      <div className="card-body" style={{ display: 'flex', flexDirection: 'column', gap: 12 }}>
+        <p style={{ fontSize: 13, color: 'var(--text-mid)', margin: 0 }}>
+          Paste a Booking.com hotel page URL to add it directly without running a discovery scrape.
+        </p>
+        <div>
+          <label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
+            Booking.com URL
+          </label>
+          <input
+            type="url"
+            placeholder="https://www.booking.com/hotel/gb/hotel-name.en-gb.html"
+            value={url}
+            onChange={e => handleUrlChange(e.target.value)}
+            style={{ width: '100%' }}
+          />
+        </div>
+        <div style={{ display: 'grid', gridTemplateColumns: '1fr auto', gap: 12 }}>
+          <div>
+            <label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
+              Display Name
+            </label>
+            <input
+              type="text"
+              placeholder="Hotel name"
+              value={name}
+              onChange={e => setName(e.target.value)}
+              style={{ width: '100%' }}
+            />
+          </div>
+          <div>
+            <label style={{ fontSize: 12, fontWeight: 600, color: 'var(--text-mid)', display: 'block', marginBottom: 4 }}>
+              Tier
+            </label>
+            <select value={tier} onChange={e => setTier(e.target.value)} style={{ width: 140 }}>
+              <option value="competitor">Competitor</option>
+              <option value="market">Market</option>
+              <option value="own">Own Hotel</option>
+            </select>
+          </div>
+        </div>
+        <div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
+          <button
+            className="btn btn-primary"
+            onClick={() => m.mutate()}
+            disabled={m.isPending || !url.trim() || !name.trim()}
+          >
+            {m.isPending ? 'Adding…' : 'Add Hotel'}
+          </button>
+          {m.isSuccess && <span style={{ fontSize: 12, color: 'var(--success)' }}>✓ Added</span>}
+          {m.isError && (
+            <span style={{ fontSize: 12, color: 'var(--danger)' }}>
+              {(m.error as any)?.response?.data?.detail || 'Failed to add hotel'}
+            </span>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+
 const HotelsTab: React.FC = () => {
   const queryClient = useQueryClient()
   const [tierFilter, setTierFilter] = useState<string>('')
@@ -1016,6 +1150,8 @@ const HotelsTab: React.FC = () => {
 
   return (
     <div>
+      <AddHotelForm />
+
       {/* Filter */}
       <div style={styles.filterRow}>
         <span style={styles.filterLabel}>Filter:</span>