From 5fbf155f62367803d0583d0802db6c98a62a9137 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Mon, 13 Jul 2026 21:10:03 +0000 Subject: [PATCH] Add weather sync job, API endpoints and settings UI Fetches daily ERA5 weather from Open-Meteo (no API key). Configurable location, timezone and sync time via Settings. Co-Authored-By: Claude Sonnet 4.6 --- backend/api/config.py | 73 +++++++++ backend/api/sync.py | 107 ++++++++++++ backend/jobs/weather_sync.py | 213 ++++++++++++++++++++++++ frontend/src/pages/Settings.tsx | 279 +++++++++++++++++++++++++++++++- 4 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 backend/jobs/weather_sync.py diff --git a/backend/api/config.py b/backend/api/config.py index 56f7765..4dccd78 100644 --- a/backend/api/config.py +++ b/backend/api/config.py @@ -1145,6 +1145,79 @@ async def delete_key( # AI INSIGHTS SETTINGS # ============================================ +class WeatherSettingsResponse(BaseModel): + enabled: bool = True + location_name: str = "Stow on the Wold, GL54 1JX" + latitude: float = 51.9253 + longitude: float = -1.7272 + timezone: str = "Europe/London" + sync_time: str = "05:15" + + +class WeatherSettingsUpdate(BaseModel): + enabled: Optional[bool] = None + location_name: Optional[str] = None + latitude: Optional[float] = None + longitude: Optional[float] = None + timezone: Optional[str] = None + sync_time: Optional[str] = None + + +@router.get("/settings/weather", response_model=WeatherSettingsResponse) +async def get_weather_settings( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Get weather sync configuration.""" + result = await db.execute( + text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'weather_%'") + ) + cfg = {row.config_key: row.config_value for row in result.fetchall()} + return WeatherSettingsResponse( + enabled=cfg.get("weather_sync_enabled", "true").lower() in ("true", "1", "yes"), + location_name=cfg.get("weather_location_name", "Stow on the Wold, GL54 1JX"), + latitude=float(cfg.get("weather_latitude", "51.9253")), + longitude=float(cfg.get("weather_longitude", "-1.7272")), + timezone=cfg.get("weather_timezone", "Europe/London"), + sync_time=cfg.get("weather_sync_time", "05:15"), + ) + + +@router.post("/settings/weather") +async def update_weather_settings( + settings: WeatherSettingsUpdate, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Update weather sync configuration.""" + updates = {} + if settings.enabled is not None: + updates["weather_sync_enabled"] = "true" if settings.enabled else "false" + if settings.location_name is not None: + updates["weather_location_name"] = settings.location_name + if settings.latitude is not None: + updates["weather_latitude"] = str(settings.latitude) + if settings.longitude is not None: + updates["weather_longitude"] = str(settings.longitude) + if settings.timezone is not None: + updates["weather_timezone"] = settings.timezone + if settings.sync_time is not None: + updates["weather_sync_time"] = settings.sync_time + + for key, value in updates.items(): + await db.execute( + text(""" + INSERT INTO system_config (config_key, config_value, updated_at) + VALUES (:key, :value, NOW()) + ON CONFLICT (config_key) DO UPDATE + SET config_value = :value, updated_at = NOW() + """), + {"key": key, "value": value} + ) + await db.commit() + return {"status": "saved", "keys_updated": list(updates.keys())} + + class AIInsightsSettingsResponse(BaseModel): enabled: bool = False api_key_set: bool = False diff --git a/backend/api/sync.py b/backend/api/sync.py index 4a8b90a..7944191 100644 --- a/backend/api/sync.py +++ b/backend/api/sync.py @@ -886,3 +886,110 @@ async def run_backfill_job( raise finally: db.close() + + +# ============================================================ +# WEATHER SYNC ENDPOINTS +# ============================================================ + +@router.post("/weather") +async def trigger_weather_sync( + background_tasks: BackgroundTasks, + from_date: Optional[date] = Query(None, description="Start date (default: 7 days ago)"), + to_date: Optional[date] = Query(None, description="End date (default: today)"), + current_user: dict = Depends(get_current_user) +): + """ + Trigger a manual weather data sync from Open-Meteo. + Defaults to last 7 days. Pass from_date/to_date for a custom range. + """ + from jobs.weather_sync import sync_weather_data + + if from_date is None: + from_date = date.today() - timedelta(days=7) + if to_date is None: + to_date = date.today() + + background_tasks.add_task( + sync_weather_data, + from_date=from_date, + to_date=to_date, + triggered_by=f"user:{current_user['username']}" + ) + + return { + "status": "started", + "source": "open_meteo", + "from_date": from_date, + "to_date": to_date, + "message": f"Weather sync {from_date} to {to_date} started in background" + } + + +@router.post("/weather/backfill") +async def trigger_weather_backfill( + background_tasks: BackgroundTasks, + from_date: date = Query(..., description="Start date for backfill"), + to_date: date = Query(..., description="End date for backfill"), + current_user: dict = Depends(get_current_user) +): + """ + Backfill weather data over an arbitrary date range. + Runs in 6-month chunks so progress is logged incrementally. + Open-Meteo has data from 1940-01-01 to present. + """ + from jobs.weather_sync import run_weather_backfill + + background_tasks.add_task( + run_weather_backfill, + from_date=from_date, + to_date=to_date, + triggered_by=f"user:{current_user['username']}" + ) + + return { + "status": "started", + "source": "open_meteo", + "from_date": from_date, + "to_date": to_date, + "message": f"Weather backfill {from_date} to {to_date} started in background" + } + + +@router.get("/weather/status") +async def get_weather_status( + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_user) +): + """Return weather data coverage and last sync status.""" + coverage = await db.execute(text(""" + SELECT + MIN(date) as first_date, + MAX(date) as last_date, + COUNT(*) as total_days + FROM weather_data + """)) + cov = coverage.fetchone() + + last_sync = await db.execute(text(""" + SELECT completed_at, status, records_fetched, records_created, date_from, date_to + FROM sync_log + WHERE source = 'open_meteo' + ORDER BY started_at DESC LIMIT 1 + """)) + sync_row = last_sync.fetchone() + + return { + "coverage": { + "first_date": cov.first_date, + "last_date": cov.last_date, + "total_days": cov.total_days, + }, + "last_sync": { + "completed_at": sync_row.completed_at if sync_row else None, + "status": sync_row.status if sync_row else None, + "records_fetched": sync_row.records_fetched if sync_row else None, + "date_from": sync_row.date_from if sync_row else None, + "date_to": sync_row.date_to if sync_row else None, + } + } diff --git a/backend/jobs/weather_sync.py b/backend/jobs/weather_sync.py new file mode 100644 index 0000000..25dadb1 --- /dev/null +++ b/backend/jobs/weather_sync.py @@ -0,0 +1,213 @@ +""" +Weather data sync - fetches daily weather from Open-Meteo archive API (ERA5). +Location: Stow on the Wold, GL54 1JX (lat=51.9253, lon=-1.7272, elev=230m). +No API key required. Stores one row per day, upserts on re-fetch. +""" +import logging +from datetime import date, timedelta +from typing import Optional + +import httpx +from dateutil.relativedelta import relativedelta +from sqlalchemy import text + +from database import SyncSessionLocal + +logger = logging.getLogger(__name__) + +OPEN_METEO_URL = "https://archive-api.open-meteo.com/v1/archive" + +# Fallback defaults — overridden by system_config at runtime +_DEFAULT_LATITUDE = 51.9253 +_DEFAULT_LONGITUDE = -1.7272 +_DEFAULT_TIMEZONE = "Europe/London" + +DAILY_VARIABLES = [ + "temperature_2m_max", + "temperature_2m_min", + "temperature_2m_mean", + "precipitation_sum", + "rain_sum", + "snowfall_sum", + "snow_depth_max", + "windspeed_10m_max", + "sunshine_duration", + "weathercode", +] + + +async def sync_weather_data( + from_date: date, + to_date: date, + triggered_by: str = "scheduler" +) -> dict: + """Fetch daily weather from Open-Meteo and upsert into weather_data.""" + logger.info(f"Starting weather sync {from_date} to {to_date}") + + db = SyncSessionLocal() + try: + # Read location config from system_config (falls back to defaults if not set) + cfg_result = db.execute( + text(""" + SELECT config_key, config_value FROM system_config + WHERE config_key IN ('weather_latitude', 'weather_longitude', 'weather_timezone') + """) + ) + cfg = {row.config_key: row.config_value for row in cfg_result.fetchall()} + latitude = float(cfg.get("weather_latitude") or _DEFAULT_LATITUDE) + longitude = float(cfg.get("weather_longitude") or _DEFAULT_LONGITUDE) + timezone = cfg.get("weather_timezone") or _DEFAULT_TIMEZONE + + db.execute( + text(""" + INSERT INTO sync_log (sync_type, source, started_at, status, date_from, date_to, triggered_by) + VALUES ('weather', 'open_meteo', NOW(), 'running', :from_date, :to_date, :triggered_by) + """), + {"from_date": from_date, "to_date": to_date, "triggered_by": triggered_by} + ) + db.commit() + + params = { + "latitude": latitude, + "longitude": longitude, + "start_date": from_date.isoformat(), + "end_date": to_date.isoformat(), + "daily": ",".join(DAILY_VARIABLES), + "timezone": timezone, + } + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(OPEN_METEO_URL, params=params) + response.raise_for_status() + data = response.json() + + daily = data.get("daily", {}) + dates = daily.get("time", []) + + if not dates: + raise Exception("No data returned from Open-Meteo API") + + records_upserted = 0 + + for i, date_str in enumerate(dates): + def val(key, idx=i): + v = daily.get(key, []) + return v[idx] if idx < len(v) else None + + sunshine_s = val("sunshine_duration") + sunshine_h = round(sunshine_s / 3600, 2) if sunshine_s is not None else None + + db.execute( + text(""" + INSERT INTO weather_data ( + date, + temperature_max, temperature_min, temperature_mean, + precipitation_sum, rain_sum, snowfall_sum, snow_depth_max, + windspeed_max, sunshine_hours, weathercode, + fetched_at + ) VALUES ( + :date, + :temp_max, :temp_min, :temp_mean, + :precip, :rain, :snowfall, :snow_depth, + :wind, :sunshine, :weathercode, + NOW() + ) + ON CONFLICT (date) DO UPDATE SET + temperature_max = :temp_max, + temperature_min = :temp_min, + temperature_mean = :temp_mean, + precipitation_sum = :precip, + rain_sum = :rain, + snowfall_sum = :snowfall, + snow_depth_max = :snow_depth, + windspeed_max = :wind, + sunshine_hours = :sunshine, + weathercode = :weathercode, + fetched_at = NOW() + """), + { + "date": date_str, + "temp_max": val("temperature_2m_max"), + "temp_min": val("temperature_2m_min"), + "temp_mean": val("temperature_2m_mean"), + "precip": val("precipitation_sum"), + "rain": val("rain_sum"), + "snowfall": val("snowfall_sum"), + "snow_depth": val("snow_depth_max"), + "wind": val("windspeed_10m_max"), + "sunshine": sunshine_h, + "weathercode": val("weathercode"), + } + ) + records_upserted += 1 + + db.commit() + + db.execute( + text(""" + UPDATE sync_log + SET completed_at = NOW(), status = 'success', + records_fetched = :fetched, records_created = :created + WHERE id = ( + SELECT id FROM sync_log + WHERE source = 'open_meteo' AND status = 'running' + ORDER BY started_at DESC LIMIT 1 + ) + """), + {"fetched": len(dates), "created": records_upserted} + ) + db.commit() + + logger.info(f"Weather sync completed: {records_upserted} days upserted") + return {"records_fetched": len(dates), "records_upserted": records_upserted} + + except Exception as e: + logger.error(f"Weather sync failed: {e}") + try: + db.rollback() + db.execute( + text(""" + UPDATE sync_log + SET completed_at = NOW(), status = 'failed', error_message = :error + WHERE id = ( + SELECT id FROM sync_log + WHERE source = 'open_meteo' AND status = 'running' + ORDER BY started_at DESC LIMIT 1 + ) + """), + {"error": str(e)[:500]} + ) + db.commit() + except Exception as log_error: + logger.error(f"Failed to update sync_log: {log_error}") + raise + finally: + db.close() + + +async def run_weather_sync(triggered_by: str = "scheduler"): + """Daily scheduled sync — last 7 days to catch ERA5 corrections.""" + from_date = date.today() - timedelta(days=7) + to_date = date.today() + await sync_weather_data(from_date, to_date, triggered_by=triggered_by) + + +async def run_weather_backfill( + from_date: date, + to_date: date, + triggered_by: str = "manual" +) -> dict: + """Backfill in 6-month chunks so progress is logged incrementally.""" + logger.info(f"Starting weather backfill {from_date} to {to_date}") + total_upserted = 0 + + chunk_start = from_date + while chunk_start <= to_date: + chunk_end = min(chunk_start + relativedelta(months=6) - timedelta(days=1), to_date) + logger.info(f"Weather backfill chunk: {chunk_start} to {chunk_end}") + result = await sync_weather_data(chunk_start, chunk_end, triggered_by=triggered_by) + total_upserted += result.get("records_upserted", 0) + chunk_start = chunk_end + timedelta(days=1) + + logger.info(f"Weather backfill complete: {total_upserted} total days") + return {"records_upserted": total_upserted} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 000cf7e..292bf07 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -80,7 +80,7 @@ const badgeStyle = (status: 'success' | 'warning' | 'error' | 'info' = 'info'): }, statuses[status]) } -type SettingsPage = 'newbook' | 'resos' | 'database' | 'special-dates' | 'budget' | 'tax-rates' | 'forecast-snapshots' | 'backup' | 'api-keys' | 'ai-insights' +type SettingsPage = 'newbook' | 'resos' | 'database' | 'special-dates' | 'budget' | 'tax-rates' | 'forecast-snapshots' | 'backup' | 'api-keys' | 'ai-insights' | 'weather' const Settings: React.FC = () => { const [activePage, setActivePage] = useState('newbook') @@ -93,6 +93,7 @@ const Settings: React.FC = () => { { id: 'tax-rates', label: 'Tax Rates' }, { id: 'forecast-snapshots', label: 'Forecast Snapshots' }, { id: 'ai-insights', label: 'AI Insights' }, + { id: 'weather', label: 'Weather' }, { id: 'api-keys', label: 'API Keys' }, { id: 'backup', label: 'Backup & Restore' }, { id: 'database', label: 'Database Browser' }, @@ -128,6 +129,7 @@ const Settings: React.FC = () => { {activePage === 'backup' && } {activePage === 'database' && } {activePage === 'ai-insights' && } + {activePage === 'weather' && } {activePage === 'api-keys' && } @@ -7235,4 +7237,279 @@ const styles: Record = { }, } +// ============================================================ +// WEATHER PAGE +// ============================================================ + +interface WeatherSettings { + enabled: boolean + location_name: string + latitude: number + longitude: number + timezone: string + sync_time: string +} + +interface WeatherStatus { + coverage: { first_date: string | null; last_date: string | null; total_days: number } + last_sync: { completed_at: string | null; status: string | null; records_fetched: number | null; date_from: string | null; date_to: string | null } +} + +const WeatherPage: React.FC = () => { + const queryClient = useQueryClient() + const [locationName, setLocationName] = useState('') + const [latitude, setLatitude] = useState('') + const [longitude, setLongitude] = useState('') + const [timezone, setTimezone] = useState('Europe/London') + const [syncTime, setSyncTime] = useState('05:15') + const [enabled, setEnabled] = useState(true) + const [saveStatus, setSaveStatus] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle') + const [syncStatus, setSyncStatus] = useState<'idle' | 'running' | 'done' | 'error'>('idle') + const [backfillFrom, setBackfillFrom] = useState('2020-01-01') + const [backfillTo, setBackfillTo] = useState(new Date().toISOString().slice(0, 10)) + const [backfillStatus, setBackfillStatus] = useState<'idle' | 'running' | 'done' | 'error'>('idle') + + const { data: settings, isLoading } = useQuery({ + queryKey: ['weather-settings'], + queryFn: async () => { + const r = await fetch('/forecasting/api/config/settings/weather') + if (!r.ok) throw new Error('Failed to load weather settings') + return r.json() + }, + }) + + const { data: status } = useQuery({ + queryKey: ['weather-status'], + queryFn: async () => { + const r = await fetch('/forecasting/api/sync/weather/status') + if (!r.ok) return null + return r.json() + }, + refetchInterval: 30000, + }) + + useEffect(() => { + if (settings) { + setLocationName(settings.location_name) + setLatitude(String(settings.latitude)) + setLongitude(String(settings.longitude)) + setTimezone(settings.timezone) + setSyncTime(settings.sync_time) + setEnabled(settings.enabled) + } + }, [settings]) + + const handleSave = async () => { + setSaveStatus('saving') + try { + const r = await fetch('/forecasting/api/config/settings/weather', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + enabled, + location_name: locationName, + latitude: parseFloat(latitude), + longitude: parseFloat(longitude), + timezone, + sync_time: syncTime, + }), + }) + if (r.ok) { + setSaveStatus('saved') + queryClient.invalidateQueries({ queryKey: ['weather-settings'] }) + setTimeout(() => setSaveStatus('idle'), 3000) + } else { + setSaveStatus('error') + setTimeout(() => setSaveStatus('idle'), 3000) + } + } catch { + setSaveStatus('error') + setTimeout(() => setSaveStatus('idle'), 3000) + } + } + + const handleSync = async () => { + setSyncStatus('running') + try { + const r = await fetch('/forecasting/api/sync/weather', { method: 'POST' }) + if (r.ok) { + setSyncStatus('done') + queryClient.invalidateQueries({ queryKey: ['weather-status'] }) + } else { + setSyncStatus('error') + } + } catch { + setSyncStatus('error') + } + setTimeout(() => setSyncStatus('idle'), 5000) + } + + const handleBackfill = async () => { + setBackfillStatus('running') + try { + const r = await fetch( + `/forecasting/api/sync/weather/backfill?from_date=${backfillFrom}&to_date=${backfillTo}`, + { method: 'POST' } + ) + if (r.ok) { + setBackfillStatus('done') + queryClient.invalidateQueries({ queryKey: ['weather-status'] }) + } else { + setBackfillStatus('error') + } + } catch { + setBackfillStatus('error') + } + setTimeout(() => setBackfillStatus('idle'), 5000) + } + + if (isLoading) return
Loading…
+ + const fieldStyle: React.CSSProperties = { + width: '100%', padding: `${spacing.sm} ${spacing.md}`, + border: `1px solid ${colors.border}`, borderRadius: radius.md, + fontSize: typography.sm, color: colors.text, background: colors.surface, + boxSizing: 'border-box', + } + const labelStyle: React.CSSProperties = { + display: 'block', fontSize: typography.sm, fontWeight: typography.medium, + color: colors.textSecondary, marginBottom: spacing.xs, + } + const rowStyle: React.CSSProperties = { marginBottom: spacing.md } + const halfRowStyle: React.CSSProperties = { display: 'grid', gridTemplateColumns: '1fr 1fr', gap: spacing.md, marginBottom: spacing.md } + + return ( +
+

+ Weather Data +

+

+ Daily weather fetched from Open-Meteo (ERA5 archive — free, no API key). + Set the coordinates for the hotel location; each stack uses its own location. +

+ + {/* Coverage status */} + {status && ( +
+
+
+
Coverage
+
+ {status.coverage.first_date ?? '—'} → {status.coverage.last_date ?? '—'} +
+
{status.coverage.total_days.toLocaleString()} days
+
+
+
Last sync
+
+ {status.last_sync.completed_at ? new Date(status.last_sync.completed_at).toLocaleString() : 'Never'} +
+ {status.last_sync.status && ( + + {status.last_sync.status} + + )} +
+
+
+ )} + + {/* Location */} +
+

Location

+ +
+ + setLocationName(e.target.value)} placeholder="e.g. Stow on the Wold, GL54 1JX" /> +
+ +
+
+ + setLatitude(e.target.value)} placeholder="51.9253" /> +
+
+ + setLongitude(e.target.value)} placeholder="-1.7272" /> +
+
+ +
+ + setTimezone(e.target.value)} placeholder="Europe/London" /> +
+
+ + {/* Schedule */} +
+

Schedule

+ +
+
+ + setSyncTime(e.target.value)} placeholder="05:15" /> +
+
+ +
+
+

+ Daily sync fetches the last 7 days — ERA5 data is updated 5 days behind real-time so recent values may be revised. +

+
+ + + + {/* Manual sync */} +
+

Manual sync

+

+ Fetches the last 7 days for the configured location. +

+ +
+ + {/* Backfill */} +
+

Backfill

+

+ Open-Meteo has ERA5 data from 1940-01-01. Use this to populate any date range. +

+
+
+ + setBackfillFrom(e.target.value)} /> +
+
+ + setBackfillTo(e.target.value)} /> +
+ +
+
+
+ ) +} + export default Settings