rates/backend/migrate_direct_data.py

"""
One-off migration: copy guestline-monitor SQLite data into PostgreSQL.
Run manually after deploy:
  docker compose exec backend python migrate_direct_data.py
"""
import json
import os
import sqlite3
import sys
from datetime import datetime, timezone

import psycopg2
import psycopg2.extras

ARCHIVE_DIR = os.environ.get(
    "GUESTLINE_ARCHIVE",
    "/home/jtr/laptop-archive/guestline-monitor/data"
)
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://rates:rates_secret@localhost:5432/rates_db")

CONFIG_DB = os.path.join(ARCHIVE_DIR, "config.db")


def dict_row(cursor, row):
    return {col[0]: val for col, val in zip(cursor.description, row)}


def migrate():
    if not os.path.exists(CONFIG_DB):
        print(f"ERROR: config.db not found at {CONFIG_DB}", file=sys.stderr)
        print("Set GUESTLINE_ARCHIVE env var to the data directory path.")
        sys.exit(1)

    pg = psycopg2.connect(DATABASE_URL)
    pg.autocommit = False

    # ── Read hotel configs from SQLite ──────────────────────────────────────
    src = sqlite3.connect(CONFIG_DB)
    src.row_factory = dict_row
    hotels = src.execute(
        """SELECT id, name, engine_profile, params, room_labels, rate_labels,
                  room_order, benchmark_room, benchmark_rate,
                  tier_base_room, tier_offsets, scrape_enabled, last_scraped_at
           FROM hotels ORDER BY id"""
    ).fetchall()
    src.close()

    if not hotels:
        print("No hotels found in config.db — nothing to migrate.")
        return

    print(f"Migrating {len(hotels)} hotels...")

    # Map old SQLite hotel IDs to new PostgreSQL IDs
    id_map: dict[int, int] = {}

    with pg.cursor() as cur:
        for h in hotels:
            params = h["params"] if isinstance(h["params"], str) else json.dumps(h["params"] or {})
            cur.execute(
                """INSERT INTO direct_competitor_hotels
                   (name, profile_name, params, room_labels, rate_labels, room_order,
                    benchmark_room, benchmark_rate, tier_base_room, tier_offsets,
                    scrape_enabled, last_scraped_at)
                   VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                   RETURNING id""",
                (
                    h["name"],
                    h["engine_profile"],
                    params,
                    h["room_labels"] or "{}",
                    h["rate_labels"] or "{}",
                    h["room_order"] or "[]",
                    h["benchmark_room"],
                    h["benchmark_rate"],
                    h["tier_base_room"],
                    h["tier_offsets"] or "{}",
                    bool(h["scrape_enabled"]),
                    h["last_scraped_at"],
                )
            )
            new_id = cur.fetchone()[0]
            id_map[h["id"]] = new_id
            print(f"  Hotel {h['id']} → {new_id}: {h['name']}")
        pg.commit()

    # ── Migrate snapshot data from per-hotel SQLite DBs ─────────────────────
    total_rows = 0
    for old_id, new_id in id_map.items():
        db_path = os.path.join(ARCHIVE_DIR, f"hotel_{old_id}.db")
        if not os.path.exists(db_path):
            print(f"  hotel_{old_id}.db not found — skipping snapshot data")
            continue

        src = sqlite3.connect(db_path)
        src.row_factory = dict_row

        # Migrate scrape_runs first
        runs = src.execute(
            "SELECT id, scraped_at, dates_found, rows_saved FROM scrape_runs ORDER BY id"
        ).fetchall()

        run_id_map: dict[int, int] = {}
        with pg.cursor() as cur:
            for run in runs:
                cur.execute(
                    """INSERT INTO direct_scrape_runs (hotel_id, scraped_at, dates_found, rows_saved)
                       VALUES (%s, %s, %s, %s) RETURNING id""",
                    (new_id, run["scraped_at"], run["dates_found"] or 0, run["rows_saved"] or 0)
                )
                run_id_map[run["id"]] = cur.fetchone()[0]
            pg.commit()

        # Migrate snapshots in batches
        BATCH = 2000
        offset = 0
        hotel_rows = 0
        while True:
            rows = src.execute(
                """SELECT scrape_run_id, scraped_at, stay_date, room_id, rate_id,
                          availability, price_excl, price_incl, currency, min_stay_nights
                   FROM snapshots ORDER BY id LIMIT ? OFFSET ?""",
                (BATCH, offset)
            ).fetchall()
            if not rows:
                break

            with pg.cursor() as cur:
                psycopg2.extras.execute_values(
                    cur,
                    """INSERT INTO direct_rates
                       (hotel_id, scrape_run_id, scraped_at, stay_date, room_id, rate_id,
                        availability, price_excl, price_incl, currency, min_stay_nights)
                       VALUES %s""",
                    [
                        (
                            new_id,
                            run_id_map.get(r["scrape_run_id"]),
                            r["scraped_at"],
                            r["stay_date"],
                            r["room_id"],
                            r["rate_id"],
                            r["availability"] or 0,
                            r["price_excl"],
                            r["price_incl"],
                            r["currency"] or "GBP",
                            r["min_stay_nights"],
                        )
                        for r in rows
                    ]
                )
                pg.commit()

            hotel_rows += len(rows)
            offset += BATCH
            print(f"    {hotel_rows} rows migrated for hotel {old_id}...", end="\r")

        print(f"  hotel_{old_id}: {hotel_rows} snapshot rows migrated")
        total_rows += hotel_rows
        src.close()

    pg.close()
    print(f"\nMigration complete: {len(hotels)} hotels, {total_rows} snapshot rows.")


if __name__ == "__main__":
    migrate()