Fix Booking.com scrape saves — widen room_type to TEXT + rollback per failure

Scraped room descriptions exceed VARCHAR(100) (StringDataRightTruncation),
and the failed insert poisoned the transaction so every subsequent save in
the batch died with InFailedSqlTransaction — scrapes reported success with
0 rows saved. Widen the column (with ALTER for existing tables) and roll
back after a failed save.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-05 14:32:20 +00:00
parent f51e1dcb76
commit 75c2adf803
2 changed files with 7 additions and 1 deletions

View file

@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS booking_com_rates (
availability_status VARCHAR(30), -- available | sold_out | no_data
rate_gross DECIMAL(10,2),
currency VARCHAR(10) DEFAULT 'GBP',
room_type VARCHAR(100),
room_type TEXT,
breakfast_included BOOLEAN DEFAULT FALSE,
free_cancellation BOOLEAN DEFAULT FALSE,
no_prepayment BOOLEAN DEFAULT FALSE,
@ -124,6 +124,9 @@ CREATE TABLE IF NOT EXISTS booking_com_rates (
scraped_at TIMESTAMPTZ DEFAULT NOW()
);
-- Widen room_type on tables created before the VARCHAR(100) → TEXT change
ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT;
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);

View file

@ -349,6 +349,9 @@ async def scrape_date(
rates_saved += 1
except Exception as e:
logger.warning(f"Error saving hotel/rate: {e}")
# A failed statement aborts the transaction — roll back so the
# remaining hotels in this batch can still be saved
db.rollback()
continue
db.commit()