From 75c2adf803d6d12e6e5649b564342ef69b61f559 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Sun, 5 Jul 2026 14:32:20 +0000 Subject: [PATCH] =?UTF-8?q?Fix=20Booking.com=20scrape=20saves=20=E2=80=94?= =?UTF-8?q?=20widen=20room=5Ftype=20to=20TEXT=20+=20rollback=20per=20failu?= =?UTF-8?q?re?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scraped room descriptions exceed VARCHAR(100) (StringDataRightTruncation), and the failed insert poisoned the transaction so every subsequent save in the batch died with InFailedSqlTransaction — scrapes reported success with 0 rows saved. Widen the column (with ALTER for existing tables) and roll back after a failed save. Co-Authored-By: Claude Fable 5 --- backend/schema.sql | 5 ++++- backend/services/booking_scraper.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/schema.sql b/backend/schema.sql index 724bde6..6515534 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS booking_com_rates ( availability_status VARCHAR(30), -- available | sold_out | no_data rate_gross DECIMAL(10,2), currency VARCHAR(10) DEFAULT 'GBP', - room_type VARCHAR(100), + room_type TEXT, breakfast_included BOOLEAN DEFAULT FALSE, free_cancellation BOOLEAN DEFAULT FALSE, no_prepayment BOOLEAN DEFAULT FALSE, @@ -124,6 +124,9 @@ CREATE TABLE IF NOT EXISTS booking_com_rates ( scraped_at TIMESTAMPTZ DEFAULT NOW() ); +-- Widen room_type on tables created before the VARCHAR(100) → TEXT change +ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT; + CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date); CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC); diff --git a/backend/services/booking_scraper.py b/backend/services/booking_scraper.py index a3bd9ce..b1217b9 100644 --- a/backend/services/booking_scraper.py +++ b/backend/services/booking_scraper.py @@ -349,6 +349,9 @@ async def scrape_date( rates_saved += 1 except Exception as e: logger.warning(f"Error saving hotel/rate: {e}") + # A failed statement aborts the transaction — roll back so the + # remaining hotels in this batch can still be saved + db.rollback() continue db.commit()