Fix Booking.com scrape saves — widen room_type to TEXT + rollback per failure
Scraped room descriptions exceed VARCHAR(100) (StringDataRightTruncation), and the failed insert poisoned the transaction so every subsequent save in the batch died with InFailedSqlTransaction — scrapes reported success with 0 rows saved. Widen the column (with ALTER for existing tables) and roll back after a failed save. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
f51e1dcb76
commit
75c2adf803
2 changed files with 7 additions and 1 deletions
|
|
@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS booking_com_rates (
|
||||||
availability_status VARCHAR(30), -- available | sold_out | no_data
|
availability_status VARCHAR(30), -- available | sold_out | no_data
|
||||||
rate_gross DECIMAL(10,2),
|
rate_gross DECIMAL(10,2),
|
||||||
currency VARCHAR(10) DEFAULT 'GBP',
|
currency VARCHAR(10) DEFAULT 'GBP',
|
||||||
room_type VARCHAR(100),
|
room_type TEXT,
|
||||||
breakfast_included BOOLEAN DEFAULT FALSE,
|
breakfast_included BOOLEAN DEFAULT FALSE,
|
||||||
free_cancellation BOOLEAN DEFAULT FALSE,
|
free_cancellation BOOLEAN DEFAULT FALSE,
|
||||||
no_prepayment BOOLEAN DEFAULT FALSE,
|
no_prepayment BOOLEAN DEFAULT FALSE,
|
||||||
|
|
@ -124,6 +124,9 @@ CREATE TABLE IF NOT EXISTS booking_com_rates (
|
||||||
scraped_at TIMESTAMPTZ DEFAULT NOW()
|
scraped_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Widen room_type on tables created before the VARCHAR(100) → TEXT change
|
||||||
|
ALTER TABLE booking_com_rates ALTER COLUMN room_type TYPE TEXT;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
|
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_hotel_date ON booking_com_rates(hotel_id, rate_date);
|
||||||
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
|
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_date ON booking_com_rates(rate_date);
|
||||||
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);
|
CREATE INDEX IF NOT EXISTS idx_booking_com_rates_scraped ON booking_com_rates(scraped_at DESC);
|
||||||
|
|
|
||||||
|
|
@ -349,6 +349,9 @@ async def scrape_date(
|
||||||
rates_saved += 1
|
rates_saved += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Error saving hotel/rate: {e}")
|
logger.warning(f"Error saving hotel/rate: {e}")
|
||||||
|
# A failed statement aborts the transaction — roll back so the
|
||||||
|
# remaining hotels in this batch can still be saved
|
||||||
|
db.rollback()
|
||||||
continue
|
continue
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue