-- Rate Monitor — Database Schema -- All CREATE TABLE/INDEX are idempotent (IF NOT EXISTS) -- ============================================ -- SYSTEM CONFIG -- ============================================ CREATE TABLE IF NOT EXISTS system_config ( config_key VARCHAR(100) PRIMARY KEY, config_value TEXT, description TEXT, updated_at TIMESTAMPTZ DEFAULT NOW(), updated_by VARCHAR(255) ); INSERT INTO system_config (config_key, config_value, description) VALUES ('booking_scraper_enabled', 'false', 'Enable automatic booking.com rate scraping (true/false)'), ('booking_scraper_paused', 'false', 'Scraper temporarily paused due to blocking (true/false)'), ('booking_scraper_pause_until', NULL, 'ISO datetime when pause expires'), ('booking_scraper_backend', 'playwright_local', 'Scraper backend: playwright_local | playwright_proxy | apify'), ('booking_scraper_daily_time', '05:30', 'Daily scrape time (HH:MM)'), ('booking_scraper_proxy_url', NULL, 'Proxy URL for playwright_proxy backend'), ('booking_scraper_proxy_username', NULL, 'Proxy username for playwright_proxy backend'), ('booking_scraper_proxy_password', NULL, 'Proxy password for playwright_proxy backend'), ('sync_newbook_current_rates_enabled', 'false', 'Enable automatic Newbook current rates sync (true/false)'), ('sync_newbook_current_rates_time', '05:20', 'Newbook current rates sync time (HH:MM)') ON CONFLICT (config_key) DO NOTHING; -- ============================================ -- BOOKING.COM SCRAPE LOG (batch tracking) -- ============================================ CREATE TABLE IF NOT EXISTS booking_scrape_log ( batch_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), scrape_type VARCHAR(50) NOT NULL, started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), completed_at TIMESTAMPTZ, status VARCHAR(20) NOT NULL DEFAULT 'running', hotels_found INTEGER DEFAULT 0, rates_scraped INTEGER DEFAULT 0, dates_queued INTEGER DEFAULT 0, dates_completed INTEGER DEFAULT 0, dates_failed INTEGER DEFAULT 0, error_message TEXT, blocked_at TIMESTAMPTZ, resume_after TIMESTAMPTZ ); CREATE INDEX IF NOT EXISTS idx_booking_scrape_log_started ON booking_scrape_log(started_at DESC); -- ============================================ -- BOOKING.COM SCRAPE CONFIG -- ============================================ CREATE TABLE IF NOT EXISTS booking_scrape_config ( id SERIAL PRIMARY KEY, location_name VARCHAR(255) NOT NULL, location_search_url TEXT, pages_to_scrape INTEGER DEFAULT 2, adults INTEGER DEFAULT 2, is_active BOOLEAN DEFAULT TRUE, created_at TIMESTAMPTZ DEFAULT NOW() ); -- ============================================ -- BOOKING.COM SCRAPE QUEUE -- ============================================ CREATE TABLE IF NOT EXISTS booking_scrape_queue ( id SERIAL PRIMARY KEY, rate_date DATE NOT NULL, status VARCHAR(20) DEFAULT 'pending', priority INTEGER DEFAULT 0, attempts INTEGER DEFAULT 0, max_attempts INTEGER DEFAULT 3, last_attempt_at TIMESTAMPTZ, completed_at TIMESTAMPTZ, created_at TIMESTAMPTZ DEFAULT NOW(), error_message TEXT, UNIQUE (rate_date, status) ); CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending ON booking_scrape_queue(status, priority DESC, rate_date ASC) WHERE status = 'pending'; -- ============================================ -- BOOKING.COM HOTELS -- ============================================ CREATE TABLE IF NOT EXISTS booking_com_hotels ( id SERIAL PRIMARY KEY, booking_com_id VARCHAR(50) UNIQUE, name VARCHAR(255) NOT NULL, booking_com_url TEXT, star_rating DECIMAL(2,1), review_score DECIMAL(3,1), review_count INTEGER DEFAULT 0, tier VARCHAR(20) DEFAULT 'market', -- own | competitor | market display_order INTEGER DEFAULT 999, notes TEXT, is_active BOOLEAN DEFAULT TRUE, first_seen_at TIMESTAMPTZ DEFAULT NOW(), last_seen_at TIMESTAMPTZ DEFAULT NOW() ); -- ============================================ -- BOOKING.COM RATES -- ============================================ CREATE TABLE IF NOT EXISTS booking_com_rates ( id SERIAL PRIMARY KEY, hotel_id INTEGER NOT NULL REFERENCES booking_com_hotels(id) ON DELETE CASCADE, rate_date DATE NOT NULL, availability_status VARCHAR(30), -- available | sold_out | no_data rate_gross DECIMAL(10,2), currency VARCHAR(10) DEFAULT 'GBP', room_type VARCHAR(100), breakfast_included BOOLEAN DEFAULT FALSE, free_cancellation BOOLEAN DEFAULT FALSE, no_prepayment BOOLEAN DEFAULT FALSE, rooms_left INTEGER, scrape_batch_id UUID REFERENCES booking_scrape_log(batch_id), scraped_at TIMESTAMPTZ DEFAULT NOW() ); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC); -- ============================================ -- RATE PARITY ALERTS -- ============================================ CREATE TABLE IF NOT EXISTS rate_parity_alerts ( id SERIAL PRIMARY KEY, rate_date DATE NOT NULL, room_category VARCHAR(255), newbook_rate DECIMAL(12,2), booking_com_rate DECIMAL(12,2), difference_pct DECIMAL(8,2), alert_type VARCHAR(20), -- higher | lower alert_status VARCHAR(20) DEFAULT 'active', -- active | acknowledged created_at TIMESTAMPTZ DEFAULT NOW(), acknowledged_at TIMESTAMPTZ, acknowledged_by VARCHAR(255), notes TEXT ); CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_date ON rate_parity_alerts(rate_date DESC); CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_status ON rate_parity_alerts(alert_status); -- ============================================ -- NEWBOOK CURRENT RATES (own hotel, for bookability display) -- ============================================ CREATE TABLE IF NOT EXISTS newbook_current_rates ( id SERIAL PRIMARY KEY, category_id VARCHAR(50) NOT NULL, rate_date DATE NOT NULL, gross_rate DECIMAL(12,2), net_rate DECIMAL(12,2), tariffs_data JSONB DEFAULT '{}', valid_from TIMESTAMP DEFAULT NOW(), last_verified_at TIMESTAMP DEFAULT NOW() ); CREATE INDEX IF NOT EXISTS idx_current_rates_date ON newbook_current_rates(rate_date); CREATE INDEX IF NOT EXISTS idx_current_rates_category ON newbook_current_rates(category_id); CREATE INDEX IF NOT EXISTS idx_current_rates_tariffs ON newbook_current_rates USING gin(tariffs_data); CREATE INDEX IF NOT EXISTS idx_current_rates_latest ON newbook_current_rates(category_id, rate_date, valid_from DESC); -- ============================================ -- NEWBOOK ROOM CATEGORIES (for bookability display) -- ============================================ CREATE TABLE IF NOT EXISTS newbook_room_categories ( id SERIAL PRIMARY KEY, site_id VARCHAR(50) NOT NULL UNIQUE, site_name VARCHAR(255) NOT NULL, site_type VARCHAR(100), room_count INTEGER DEFAULT 0, is_included BOOLEAN DEFAULT TRUE, display_order INTEGER DEFAULT 0, fetched_at TIMESTAMP DEFAULT NOW() ); -- ============================================ -- NEWBOOK OCCUPANCY REPORT DATA -- ============================================ CREATE TABLE IF NOT EXISTS newbook_occupancy_report_data ( id SERIAL PRIMARY KEY, date DATE NOT NULL, category_id VARCHAR(50) NOT NULL, category_name VARCHAR(255), available INTEGER DEFAULT 0, occupied INTEGER DEFAULT 0, maintenance INTEGER DEFAULT 0, allotted INTEGER DEFAULT 0, revenue_gross DECIMAL(12,2) DEFAULT 0, revenue_net DECIMAL(12,2) DEFAULT 0, occupancy_pct DECIMAL(5,2), fetched_at TIMESTAMP DEFAULT NOW(), UNIQUE(date, category_id) ); CREATE INDEX IF NOT EXISTS idx_occupancy_report_date ON newbook_occupancy_report_data(date); -- ============================================ -- DIRECT COMPETITOR HOTEL CONFIGS -- ============================================ CREATE TABLE IF NOT EXISTS direct_competitor_hotels ( id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, profile_name VARCHAR(50) NOT NULL, params JSONB NOT NULL DEFAULT '{}', room_labels JSONB DEFAULT '{}', rate_labels JSONB DEFAULT '{}', room_order JSONB DEFAULT '[]', benchmark_room VARCHAR(100), benchmark_rate VARCHAR(100), tier_base_room VARCHAR(100), tier_offsets JSONB DEFAULT '{}', scrape_enabled BOOLEAN DEFAULT TRUE, last_scraped_at TIMESTAMPTZ, created_at TIMESTAMPTZ DEFAULT NOW() ); -- ============================================ -- DIRECT SCRAPE RUNS -- ============================================ CREATE TABLE IF NOT EXISTS direct_scrape_runs ( id SERIAL PRIMARY KEY, hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE, scraped_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), dates_found INTEGER DEFAULT 0, rows_saved INTEGER DEFAULT 0 ); -- ============================================ -- DIRECT RATES (per-room snapshots from booking engines) -- ============================================ CREATE TABLE IF NOT EXISTS direct_rates ( id SERIAL PRIMARY KEY, hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE, scrape_run_id INTEGER REFERENCES direct_scrape_runs(id), scraped_at TIMESTAMPTZ NOT NULL, stay_date DATE NOT NULL, room_id VARCHAR(100) NOT NULL, rate_id VARCHAR(100) NOT NULL, availability INTEGER DEFAULT 0, price_excl DECIMAL(10,2), price_incl DECIMAL(10,2), currency VARCHAR(10) DEFAULT 'GBP', min_stay_nights INTEGER ); CREATE INDEX IF NOT EXISTS idx_direct_rates_hotel_date ON direct_rates(hotel_id, stay_date); CREATE INDEX IF NOT EXISTS idx_direct_rates_scraped ON direct_rates(scraped_at DESC); -- Link booking_com_hotels to direct_competitor_hotels (optional, for Market View direct column) ALTER TABLE booking_com_hotels ADD COLUMN IF NOT EXISTS direct_hotel_id INTEGER REFERENCES direct_competitor_hotels(id) ON DELETE SET NULL;