""" One-off migration: copy guestline-monitor SQLite data into PostgreSQL. Run manually after deploy: docker compose exec backend python migrate_direct_data.py """ import json import os import sqlite3 import sys from datetime import datetime, timezone import psycopg2 ARCHIVE_DIR = os.environ.get( "GUESTLINE_ARCHIVE", "/home/jtr/laptop-archive/guestline-monitor/data" ) DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://rates:rates_secret@localhost:5432/rates_db") CONFIG_DB = os.path.join(ARCHIVE_DIR, "config.db") def dict_row(cursor, row): return {col[0]: val for col, val in zip(cursor.description, row)} def migrate(): if not os.path.exists(CONFIG_DB): print(f"ERROR: config.db not found at {CONFIG_DB}", file=sys.stderr) print("Set GUESTLINE_ARCHIVE env var to the data directory path.") sys.exit(1) pg = psycopg2.connect(DATABASE_URL) pg.autocommit = False # ── Read hotel configs from SQLite ────────────────────────────────────── src = sqlite3.connect(CONFIG_DB) src.row_factory = dict_row hotels = src.execute( """SELECT id, name, profile, params, room_labels, rate_labels, room_order, benchmark_room, benchmark_rate, tier_base_room, tier_offsets, scrape_enabled, last_scraped_at FROM hotels ORDER BY id""" ).fetchall() src.close() if not hotels: print("No hotels found in config.db — nothing to migrate.") return print(f"Migrating {len(hotels)} hotels...") # Map old SQLite hotel IDs to new PostgreSQL IDs id_map: dict[int, int] = {} with pg.cursor() as cur: for h in hotels: params = h["params"] if isinstance(h["params"], str) else json.dumps(h["params"] or {}) cur.execute( """INSERT INTO direct_competitor_hotels (name, profile_name, params, room_labels, rate_labels, room_order, benchmark_room, benchmark_rate, tier_base_room, tier_offsets, scrape_enabled, last_scraped_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""", ( h["name"], h["profile"], params, h["room_labels"] or "{}", h["rate_labels"] or "{}", h["room_order"] or "[]", h["benchmark_room"], h["benchmark_rate"], h["tier_base_room"], h["tier_offsets"] or "{}", bool(h["scrape_enabled"]), h["last_scraped_at"], ) ) new_id = cur.fetchone()[0] id_map[h["id"]] = new_id print(f" Hotel {h['id']} → {new_id}: {h['name']}") pg.commit() # ── Migrate snapshot data from per-hotel SQLite DBs ───────────────────── total_rows = 0 for old_id, new_id in id_map.items(): db_path = os.path.join(ARCHIVE_DIR, f"hotel_{old_id}.db") if not os.path.exists(db_path): print(f" hotel_{old_id}.db not found — skipping snapshot data") continue src = sqlite3.connect(db_path) src.row_factory = dict_row # Migrate scrape_runs first runs = src.execute( "SELECT id, scraped_at, dates_found, rows_saved FROM scrape_runs ORDER BY id" ).fetchall() run_id_map: dict[int, int] = {} with pg.cursor() as cur: for run in runs: cur.execute( """INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found, rows_saved) VALUES (%s, %s, %s, %s) RETURNING id""", (new_id, run["scraped_at"], run["dates_found"] or 0, run["rows_saved"] or 0) ) run_id_map[run["id"]] = cur.fetchone()[0] pg.commit() # Migrate snapshots in batches BATCH = 2000 offset = 0 hotel_rows = 0 while True: rows = src.execute( """SELECT scrape_run_id, scraped_at, stay_date, room_id, rate_id, availability, price_excl, price_incl, currency, min_stay_nights FROM snapshots ORDER BY id LIMIT ? OFFSET ?""", (BATCH, offset) ).fetchall() if not rows: break with pg.cursor() as cur: psycopg2.extras.execute_values( cur, """INSERT INTO direct_rates (hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id, availability, price_excl, price_incl, currency, min_stay_nights) VALUES %s""", [ ( new_id, run_id_map.get(r["scrape_run_id"]), r["scraped_at"], r["stay_date"], r["room_id"], r["rate_id"], r["availability"] or 0, r["price_excl"], r["price_incl"], r["currency"] or "GBP", r["min_stay_nights"], ) for r in rows ] ) pg.commit() hotel_rows += len(rows) offset += BATCH print(f" {hotel_rows} rows migrated for hotel {old_id}...", end="\r") print(f" hotel_{old_id}: {hotel_rows} snapshot rows migrated") total_rows += hotel_rows src.close() pg.close() print(f"\nMigration complete: {len(hotels)} hotels, {total_rows} snapshot rows.") if __name__ == "__main__": import psycopg2.extras migrate()