jtricerolph
2743b0e877
Schema: migrate newbook_occupancy_report_data from single-row upsert to
snapshot model (drop unique constraint, add valid_from / last_verified_at)
matching the pattern used by newbook_current_rates.
Backend: sync_occupancy now inserts a new row only when occupied/available/
maintenance figures change, otherwise bumps last_verified_at. New endpoint
GET /bookability/occupancy-history/{category_id}/{date} returns the timeline.
Rate matrix query updated to DISTINCT ON for the multi-row table.
Frontend: clicking any cell in the Bookability matrix opens a modal with
two stacked Plotly charts — rate history per tariff (step lines, green/red
markers for available/unavailable) and occupancy pick-up over time.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
325 lines
13 KiB
SQL
325 lines
13 KiB
SQL
-- Rate Monitor — Database Schema
|
|
-- All CREATE TABLE/INDEX are idempotent (IF NOT EXISTS)
|
|
|
|
-- ============================================
|
|
-- SYSTEM CONFIG
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS system_config (
|
|
config_key VARCHAR(100) PRIMARY KEY,
|
|
config_value TEXT,
|
|
description TEXT,
|
|
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_by VARCHAR(255)
|
|
);
|
|
|
|
INSERT INTO system_config (config_key, config_value, description) VALUES
|
|
('booking_scraper_enabled', 'false', 'Enable automatic booking.com rate scraping (true/false)'),
|
|
('booking_scraper_paused', 'false', 'Scraper temporarily paused due to blocking (true/false)'),
|
|
('booking_scraper_pause_until', NULL, 'ISO datetime when pause expires'),
|
|
('booking_scraper_backend', 'playwright_local', 'Scraper backend: playwright_local | playwright_hotel_page | apify'),
|
|
('booking_scraper_daily_time', '05:30', 'Daily scrape time (HH:MM)'),
|
|
('booking_scraper_proxy_url', NULL, 'Proxy URL for playwright_proxy backend'),
|
|
('booking_scraper_proxy_username', NULL, 'Proxy username for playwright_proxy backend'),
|
|
('booking_scraper_proxy_password', NULL, 'Proxy password for playwright_proxy backend'),
|
|
('sync_newbook_current_rates_enabled', 'false', 'Enable automatic Newbook current rates sync (true/false)'),
|
|
('sync_newbook_current_rates_time', '05:20', 'Newbook current rates sync time (HH:MM)')
|
|
ON CONFLICT (config_key) DO NOTHING;
|
|
|
|
-- ============================================
|
|
-- BOOKING.COM SCRAPE LOG (batch tracking)
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS booking_scrape_log (
|
|
batch_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
scrape_type VARCHAR(50) NOT NULL,
|
|
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
completed_at TIMESTAMPTZ,
|
|
status VARCHAR(20) NOT NULL DEFAULT 'running',
|
|
hotels_found INTEGER DEFAULT 0,
|
|
rates_scraped INTEGER DEFAULT 0,
|
|
dates_queued INTEGER DEFAULT 0,
|
|
dates_completed INTEGER DEFAULT 0,
|
|
dates_failed INTEGER DEFAULT 0,
|
|
error_message TEXT,
|
|
blocked_at TIMESTAMPTZ,
|
|
resume_after TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_booking_scrape_log_started ON booking_scrape_log(started_at DESC);
|
|
|
|
-- ============================================
|
|
-- BOOKING.COM SCRAPE CONFIG
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS booking_scrape_config (
|
|
id SERIAL PRIMARY KEY,
|
|
location_name VARCHAR(255) NOT NULL,
|
|
location_search_url TEXT,
|
|
pages_to_scrape INTEGER DEFAULT 2,
|
|
adults INTEGER DEFAULT 2,
|
|
is_active BOOLEAN DEFAULT TRUE,
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- ============================================
|
|
-- BOOKING.COM SCRAPE QUEUE
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS booking_scrape_queue (
|
|
id SERIAL PRIMARY KEY,
|
|
rate_date DATE NOT NULL,
|
|
status VARCHAR(20) DEFAULT 'pending',
|
|
priority INTEGER DEFAULT 0,
|
|
attempts INTEGER DEFAULT 0,
|
|
max_attempts INTEGER DEFAULT 3,
|
|
last_attempt_at TIMESTAMPTZ,
|
|
completed_at TIMESTAMPTZ,
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
error_message TEXT,
|
|
UNIQUE (rate_date, status)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_booking_scrape_queue_pending
|
|
ON booking_scrape_queue(status, priority DESC, rate_date ASC)
|
|
WHERE status = 'pending';
|
|
|
|
-- ============================================
|
|
-- BOOKING.COM HOTELS
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS booking_com_hotels (
|
|
id SERIAL PRIMARY KEY,
|
|
booking_com_id VARCHAR(255) UNIQUE,
|
|
name VARCHAR(255) NOT NULL,
|
|
booking_com_url TEXT,
|
|
star_rating DECIMAL(2,1),
|
|
review_score DECIMAL(3,1),
|
|
review_count INTEGER DEFAULT 0,
|
|
tier VARCHAR(20) DEFAULT 'market', -- own | competitor | market
|
|
display_order INTEGER DEFAULT 999,
|
|
notes TEXT,
|
|
is_active BOOLEAN DEFAULT TRUE,
|
|
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
|
last_seen_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- ============================================
|
|
-- BOOKING.COM RATES
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS booking_com_rates (
|
|
id SERIAL PRIMARY KEY,
|
|
hotel_id INTEGER NOT NULL REFERENCES booking_com_hotels(id) ON DELETE CASCADE,
|
|
rate_date DATE NOT NULL,
|
|
availability_status VARCHAR(30), -- available | sold_out | no_data
|
|
rate_gross DECIMAL(10,2),
|
|
currency VARCHAR(10) DEFAULT 'GBP',
|
|
room_type TEXT,
|
|
breakfast_included BOOLEAN DEFAULT FALSE,
|
|
free_cancellation BOOLEAN DEFAULT FALSE,
|
|
no_prepayment BOOLEAN DEFAULT FALSE,
|
|
rooms_left INTEGER,
|
|
scrape_batch_id UUID REFERENCES booking_scrape_log(batch_id),
|
|
scraped_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- Widen columns on tables created before the type changes
|
|
ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT;
|
|
ALTER TABLE booking_com_hotels ALTER COLUMN booking_com_id TYPE VARCHAR(255);
|
|
|
|
-- Hotel page scraper: rate plan identifier and occupancy per row
|
|
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS rate_plan_id TEXT;
|
|
ALTER TABLE booking_com_rates ADD COLUMN IF NOT EXISTS max_persons INTEGER;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
|
|
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
|
|
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);
|
|
|
|
-- ============================================
|
|
-- RATE PARITY ALERTS
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS rate_parity_alerts (
|
|
id SERIAL PRIMARY KEY,
|
|
rate_date DATE NOT NULL,
|
|
room_category VARCHAR(255),
|
|
newbook_rate DECIMAL(12,2),
|
|
booking_com_rate DECIMAL(12,2),
|
|
difference_pct DECIMAL(8,2),
|
|
alert_type VARCHAR(20), -- higher | lower
|
|
alert_status VARCHAR(20) DEFAULT 'active', -- active | acknowledged
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
acknowledged_at TIMESTAMPTZ,
|
|
acknowledged_by VARCHAR(255),
|
|
notes TEXT
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_date ON rate_parity_alerts(rate_date DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_rate_parity_alerts_status ON rate_parity_alerts(alert_status);
|
|
|
|
-- ============================================
|
|
-- NEWBOOK CURRENT RATES (own hotel, for bookability display)
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS newbook_current_rates (
|
|
id SERIAL PRIMARY KEY,
|
|
category_id VARCHAR(50) NOT NULL,
|
|
rate_date DATE NOT NULL,
|
|
rate_gross DECIMAL(12,2),
|
|
rate_net DECIMAL(12,2),
|
|
tariffs_data JSONB DEFAULT '{}',
|
|
valid_from TIMESTAMP DEFAULT NOW(),
|
|
last_verified_at TIMESTAMP DEFAULT NOW()
|
|
);
|
|
|
|
-- Migrate tables created before the gross_rate/net_rate → rate_gross/rate_net rename
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (SELECT 1 FROM information_schema.columns
|
|
WHERE table_name = 'newbook_current_rates' AND column_name = 'gross_rate') THEN
|
|
ALTER TABLE newbook_current_rates RENAME COLUMN gross_rate TO rate_gross;
|
|
ALTER TABLE newbook_current_rates RENAME COLUMN net_rate TO rate_net;
|
|
END IF;
|
|
END $$;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_current_rates_date ON newbook_current_rates(rate_date);
|
|
CREATE INDEX IF NOT EXISTS idx_current_rates_category ON newbook_current_rates(category_id);
|
|
CREATE INDEX IF NOT EXISTS idx_current_rates_tariffs ON newbook_current_rates USING gin(tariffs_data);
|
|
CREATE INDEX IF NOT EXISTS idx_current_rates_latest ON newbook_current_rates(category_id, rate_date, valid_from DESC);
|
|
|
|
-- ============================================
|
|
-- NEWBOOK ROOM CATEGORIES (for bookability display)
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS newbook_room_categories (
|
|
id SERIAL PRIMARY KEY,
|
|
site_id VARCHAR(50) NOT NULL UNIQUE,
|
|
site_name VARCHAR(255) NOT NULL,
|
|
site_type VARCHAR(100),
|
|
room_count INTEGER DEFAULT 0,
|
|
is_included BOOLEAN DEFAULT TRUE,
|
|
display_order INTEGER DEFAULT 0,
|
|
fetched_at TIMESTAMP DEFAULT NOW()
|
|
);
|
|
|
|
-- ============================================
|
|
-- NEWBOOK OCCUPANCY REPORT DATA
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS newbook_occupancy_report_data (
|
|
id SERIAL PRIMARY KEY,
|
|
date DATE NOT NULL,
|
|
category_id VARCHAR(50) NOT NULL,
|
|
category_name VARCHAR(255),
|
|
available INTEGER DEFAULT 0,
|
|
occupied INTEGER DEFAULT 0,
|
|
maintenance INTEGER DEFAULT 0,
|
|
allotted INTEGER DEFAULT 0,
|
|
revenue_gross DECIMAL(12,2) DEFAULT 0,
|
|
revenue_net DECIMAL(12,2) DEFAULT 0,
|
|
occupancy_pct DECIMAL(5,2),
|
|
fetched_at TIMESTAMP DEFAULT NOW(),
|
|
UNIQUE(date, category_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_occupancy_report_date ON newbook_occupancy_report_data(date);
|
|
|
|
-- Migrate occupancy table to snapshot model (drop unique constraint, add valid_from / last_verified_at)
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM pg_constraint c
|
|
JOIN pg_class t ON t.oid = c.conrelid
|
|
WHERE t.relname = 'newbook_occupancy_report_data'
|
|
AND c.contype = 'u'
|
|
AND c.conname = 'newbook_occupancy_report_data_date_category_id_key'
|
|
) THEN
|
|
ALTER TABLE newbook_occupancy_report_data
|
|
DROP CONSTRAINT newbook_occupancy_report_data_date_category_id_key;
|
|
END IF;
|
|
|
|
IF NOT EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_name = 'newbook_occupancy_report_data' AND column_name = 'valid_from'
|
|
) THEN
|
|
ALTER TABLE newbook_occupancy_report_data ADD COLUMN valid_from TIMESTAMPTZ;
|
|
UPDATE newbook_occupancy_report_data SET valid_from = fetched_at WHERE valid_from IS NULL;
|
|
ALTER TABLE newbook_occupancy_report_data ALTER COLUMN valid_from SET DEFAULT NOW();
|
|
END IF;
|
|
|
|
IF NOT EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_name = 'newbook_occupancy_report_data' AND column_name = 'last_verified_at'
|
|
) THEN
|
|
ALTER TABLE newbook_occupancy_report_data ADD COLUMN last_verified_at TIMESTAMPTZ;
|
|
UPDATE newbook_occupancy_report_data SET last_verified_at = fetched_at WHERE last_verified_at IS NULL;
|
|
ALTER TABLE newbook_occupancy_report_data ALTER COLUMN last_verified_at SET DEFAULT NOW();
|
|
END IF;
|
|
END $$;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_occupancy_report_latest
|
|
ON newbook_occupancy_report_data(date, category_id, valid_from DESC);
|
|
|
|
-- ============================================
|
|
-- DIRECT COMPETITOR HOTEL CONFIGS
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS direct_competitor_hotels (
|
|
id SERIAL PRIMARY KEY,
|
|
name VARCHAR(255) NOT NULL,
|
|
profile_name VARCHAR(50) NOT NULL,
|
|
params JSONB NOT NULL DEFAULT '{}',
|
|
room_labels JSONB DEFAULT '{}',
|
|
rate_labels JSONB DEFAULT '{}',
|
|
room_order JSONB DEFAULT '[]',
|
|
benchmark_room VARCHAR(100),
|
|
benchmark_rate VARCHAR(100),
|
|
tier_base_room VARCHAR(100),
|
|
tier_offsets JSONB DEFAULT '{}',
|
|
scrape_enabled BOOLEAN DEFAULT TRUE,
|
|
last_scraped_at TIMESTAMPTZ,
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- ============================================
|
|
-- DIRECT SCRAPE RUNS
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS direct_scrape_runs (
|
|
id SERIAL PRIMARY KEY,
|
|
hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE,
|
|
scraped_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
dates_found INTEGER DEFAULT 0,
|
|
rows_saved INTEGER DEFAULT 0
|
|
);
|
|
|
|
-- ============================================
|
|
-- DIRECT RATES (per-room snapshots from booking engines)
|
|
-- ============================================
|
|
|
|
CREATE TABLE IF NOT EXISTS direct_rates (
|
|
id SERIAL PRIMARY KEY,
|
|
hotel_id INTEGER NOT NULL REFERENCES direct_competitor_hotels(id) ON DELETE CASCADE,
|
|
scrape_run_id INTEGER REFERENCES direct_scrape_runs(id),
|
|
scraped_at TIMESTAMPTZ NOT NULL,
|
|
stay_date DATE NOT NULL,
|
|
room_id VARCHAR(100) NOT NULL,
|
|
rate_id VARCHAR(100) NOT NULL,
|
|
availability INTEGER DEFAULT 0,
|
|
price_excl DECIMAL(10,2),
|
|
price_incl DECIMAL(10,2),
|
|
currency VARCHAR(10) DEFAULT 'GBP',
|
|
min_stay_nights INTEGER
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_direct_rates_hotel_date ON direct_rates(hotel_id, stay_date);
|
|
CREATE INDEX IF NOT EXISTS idx_direct_rates_scraped ON direct_rates(scraped_at DESC);
|
|
|
|
-- Link booking_com_hotels to direct_competitor_hotels (optional, for Market View direct column)
|
|
ALTER TABLE booking_com_hotels ADD COLUMN IF NOT EXISTS direct_hotel_id INTEGER
|
|
REFERENCES direct_competitor_hotels(id) ON DELETE SET NULL;
|
|
|
|
-- Pin the Booking.com destination: free-text ss= searches non-deterministically
|
|
-- resolve to the wrong place (Stow on the Wold once matched St. Wolfgang, AT).
|
|
ALTER TABLE booking_scrape_config ADD COLUMN IF NOT EXISTS dest_id VARCHAR(32);
|