Add weather sync job, API endpoints and settings UI
Fetches daily ERA5 weather from Open-Meteo (no API key). Configurable location, timezone and sync time via Settings. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
fc15bc5d9a
commit
5fbf155f62
4 changed files with 671 additions and 1 deletions
|
|
@ -1145,6 +1145,79 @@ async def delete_key(
|
|||
# AI INSIGHTS SETTINGS
|
||||
# ============================================
|
||||
|
||||
class WeatherSettingsResponse(BaseModel):
|
||||
enabled: bool = True
|
||||
location_name: str = "Stow on the Wold, GL54 1JX"
|
||||
latitude: float = 51.9253
|
||||
longitude: float = -1.7272
|
||||
timezone: str = "Europe/London"
|
||||
sync_time: str = "05:15"
|
||||
|
||||
|
||||
class WeatherSettingsUpdate(BaseModel):
|
||||
enabled: Optional[bool] = None
|
||||
location_name: Optional[str] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
timezone: Optional[str] = None
|
||||
sync_time: Optional[str] = None
|
||||
|
||||
|
||||
@router.get("/settings/weather", response_model=WeatherSettingsResponse)
|
||||
async def get_weather_settings(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Get weather sync configuration."""
|
||||
result = await db.execute(
|
||||
text("SELECT config_key, config_value FROM system_config WHERE config_key LIKE 'weather_%'")
|
||||
)
|
||||
cfg = {row.config_key: row.config_value for row in result.fetchall()}
|
||||
return WeatherSettingsResponse(
|
||||
enabled=cfg.get("weather_sync_enabled", "true").lower() in ("true", "1", "yes"),
|
||||
location_name=cfg.get("weather_location_name", "Stow on the Wold, GL54 1JX"),
|
||||
latitude=float(cfg.get("weather_latitude", "51.9253")),
|
||||
longitude=float(cfg.get("weather_longitude", "-1.7272")),
|
||||
timezone=cfg.get("weather_timezone", "Europe/London"),
|
||||
sync_time=cfg.get("weather_sync_time", "05:15"),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/settings/weather")
|
||||
async def update_weather_settings(
|
||||
settings: WeatherSettingsUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Update weather sync configuration."""
|
||||
updates = {}
|
||||
if settings.enabled is not None:
|
||||
updates["weather_sync_enabled"] = "true" if settings.enabled else "false"
|
||||
if settings.location_name is not None:
|
||||
updates["weather_location_name"] = settings.location_name
|
||||
if settings.latitude is not None:
|
||||
updates["weather_latitude"] = str(settings.latitude)
|
||||
if settings.longitude is not None:
|
||||
updates["weather_longitude"] = str(settings.longitude)
|
||||
if settings.timezone is not None:
|
||||
updates["weather_timezone"] = settings.timezone
|
||||
if settings.sync_time is not None:
|
||||
updates["weather_sync_time"] = settings.sync_time
|
||||
|
||||
for key, value in updates.items():
|
||||
await db.execute(
|
||||
text("""
|
||||
INSERT INTO system_config (config_key, config_value, updated_at)
|
||||
VALUES (:key, :value, NOW())
|
||||
ON CONFLICT (config_key) DO UPDATE
|
||||
SET config_value = :value, updated_at = NOW()
|
||||
"""),
|
||||
{"key": key, "value": value}
|
||||
)
|
||||
await db.commit()
|
||||
return {"status": "saved", "keys_updated": list(updates.keys())}
|
||||
|
||||
|
||||
class AIInsightsSettingsResponse(BaseModel):
|
||||
enabled: bool = False
|
||||
api_key_set: bool = False
|
||||
|
|
|
|||
|
|
@ -886,3 +886,110 @@ async def run_backfill_job(
|
|||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# WEATHER SYNC ENDPOINTS
|
||||
# ============================================================
|
||||
|
||||
@router.post("/weather")
|
||||
async def trigger_weather_sync(
|
||||
background_tasks: BackgroundTasks,
|
||||
from_date: Optional[date] = Query(None, description="Start date (default: 7 days ago)"),
|
||||
to_date: Optional[date] = Query(None, description="End date (default: today)"),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Trigger a manual weather data sync from Open-Meteo.
|
||||
Defaults to last 7 days. Pass from_date/to_date for a custom range.
|
||||
"""
|
||||
from jobs.weather_sync import sync_weather_data
|
||||
|
||||
if from_date is None:
|
||||
from_date = date.today() - timedelta(days=7)
|
||||
if to_date is None:
|
||||
to_date = date.today()
|
||||
|
||||
background_tasks.add_task(
|
||||
sync_weather_data,
|
||||
from_date=from_date,
|
||||
to_date=to_date,
|
||||
triggered_by=f"user:{current_user['username']}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "started",
|
||||
"source": "open_meteo",
|
||||
"from_date": from_date,
|
||||
"to_date": to_date,
|
||||
"message": f"Weather sync {from_date} to {to_date} started in background"
|
||||
}
|
||||
|
||||
|
||||
@router.post("/weather/backfill")
|
||||
async def trigger_weather_backfill(
|
||||
background_tasks: BackgroundTasks,
|
||||
from_date: date = Query(..., description="Start date for backfill"),
|
||||
to_date: date = Query(..., description="End date for backfill"),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Backfill weather data over an arbitrary date range.
|
||||
Runs in 6-month chunks so progress is logged incrementally.
|
||||
Open-Meteo has data from 1940-01-01 to present.
|
||||
"""
|
||||
from jobs.weather_sync import run_weather_backfill
|
||||
|
||||
background_tasks.add_task(
|
||||
run_weather_backfill,
|
||||
from_date=from_date,
|
||||
to_date=to_date,
|
||||
triggered_by=f"user:{current_user['username']}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "started",
|
||||
"source": "open_meteo",
|
||||
"from_date": from_date,
|
||||
"to_date": to_date,
|
||||
"message": f"Weather backfill {from_date} to {to_date} started in background"
|
||||
}
|
||||
|
||||
|
||||
@router.get("/weather/status")
|
||||
async def get_weather_status(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user)
|
||||
):
|
||||
"""Return weather data coverage and last sync status."""
|
||||
coverage = await db.execute(text("""
|
||||
SELECT
|
||||
MIN(date) as first_date,
|
||||
MAX(date) as last_date,
|
||||
COUNT(*) as total_days
|
||||
FROM weather_data
|
||||
"""))
|
||||
cov = coverage.fetchone()
|
||||
|
||||
last_sync = await db.execute(text("""
|
||||
SELECT completed_at, status, records_fetched, records_created, date_from, date_to
|
||||
FROM sync_log
|
||||
WHERE source = 'open_meteo'
|
||||
ORDER BY started_at DESC LIMIT 1
|
||||
"""))
|
||||
sync_row = last_sync.fetchone()
|
||||
|
||||
return {
|
||||
"coverage": {
|
||||
"first_date": cov.first_date,
|
||||
"last_date": cov.last_date,
|
||||
"total_days": cov.total_days,
|
||||
},
|
||||
"last_sync": {
|
||||
"completed_at": sync_row.completed_at if sync_row else None,
|
||||
"status": sync_row.status if sync_row else None,
|
||||
"records_fetched": sync_row.records_fetched if sync_row else None,
|
||||
"date_from": sync_row.date_from if sync_row else None,
|
||||
"date_to": sync_row.date_to if sync_row else None,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
213
backend/jobs/weather_sync.py
Normal file
213
backend/jobs/weather_sync.py
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
"""
|
||||
Weather data sync - fetches daily weather from Open-Meteo archive API (ERA5).
|
||||
Location: Stow on the Wold, GL54 1JX (lat=51.9253, lon=-1.7272, elev=230m).
|
||||
No API key required. Stores one row per day, upserts on re-fetch.
|
||||
"""
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from sqlalchemy import text
|
||||
|
||||
from database import SyncSessionLocal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPEN_METEO_URL = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
# Fallback defaults — overridden by system_config at runtime
|
||||
_DEFAULT_LATITUDE = 51.9253
|
||||
_DEFAULT_LONGITUDE = -1.7272
|
||||
_DEFAULT_TIMEZONE = "Europe/London"
|
||||
|
||||
DAILY_VARIABLES = [
|
||||
"temperature_2m_max",
|
||||
"temperature_2m_min",
|
||||
"temperature_2m_mean",
|
||||
"precipitation_sum",
|
||||
"rain_sum",
|
||||
"snowfall_sum",
|
||||
"snow_depth_max",
|
||||
"windspeed_10m_max",
|
||||
"sunshine_duration",
|
||||
"weathercode",
|
||||
]
|
||||
|
||||
|
||||
async def sync_weather_data(
|
||||
from_date: date,
|
||||
to_date: date,
|
||||
triggered_by: str = "scheduler"
|
||||
) -> dict:
|
||||
"""Fetch daily weather from Open-Meteo and upsert into weather_data."""
|
||||
logger.info(f"Starting weather sync {from_date} to {to_date}")
|
||||
|
||||
db = SyncSessionLocal()
|
||||
try:
|
||||
# Read location config from system_config (falls back to defaults if not set)
|
||||
cfg_result = db.execute(
|
||||
text("""
|
||||
SELECT config_key, config_value FROM system_config
|
||||
WHERE config_key IN ('weather_latitude', 'weather_longitude', 'weather_timezone')
|
||||
""")
|
||||
)
|
||||
cfg = {row.config_key: row.config_value for row in cfg_result.fetchall()}
|
||||
latitude = float(cfg.get("weather_latitude") or _DEFAULT_LATITUDE)
|
||||
longitude = float(cfg.get("weather_longitude") or _DEFAULT_LONGITUDE)
|
||||
timezone = cfg.get("weather_timezone") or _DEFAULT_TIMEZONE
|
||||
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO sync_log (sync_type, source, started_at, status, date_from, date_to, triggered_by)
|
||||
VALUES ('weather', 'open_meteo', NOW(), 'running', :from_date, :to_date, :triggered_by)
|
||||
"""),
|
||||
{"from_date": from_date, "to_date": to_date, "triggered_by": triggered_by}
|
||||
)
|
||||
db.commit()
|
||||
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"start_date": from_date.isoformat(),
|
||||
"end_date": to_date.isoformat(),
|
||||
"daily": ",".join(DAILY_VARIABLES),
|
||||
"timezone": timezone,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.get(OPEN_METEO_URL, params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
daily = data.get("daily", {})
|
||||
dates = daily.get("time", [])
|
||||
|
||||
if not dates:
|
||||
raise Exception("No data returned from Open-Meteo API")
|
||||
|
||||
records_upserted = 0
|
||||
|
||||
for i, date_str in enumerate(dates):
|
||||
def val(key, idx=i):
|
||||
v = daily.get(key, [])
|
||||
return v[idx] if idx < len(v) else None
|
||||
|
||||
sunshine_s = val("sunshine_duration")
|
||||
sunshine_h = round(sunshine_s / 3600, 2) if sunshine_s is not None else None
|
||||
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO weather_data (
|
||||
date,
|
||||
temperature_max, temperature_min, temperature_mean,
|
||||
precipitation_sum, rain_sum, snowfall_sum, snow_depth_max,
|
||||
windspeed_max, sunshine_hours, weathercode,
|
||||
fetched_at
|
||||
) VALUES (
|
||||
:date,
|
||||
:temp_max, :temp_min, :temp_mean,
|
||||
:precip, :rain, :snowfall, :snow_depth,
|
||||
:wind, :sunshine, :weathercode,
|
||||
NOW()
|
||||
)
|
||||
ON CONFLICT (date) DO UPDATE SET
|
||||
temperature_max = :temp_max,
|
||||
temperature_min = :temp_min,
|
||||
temperature_mean = :temp_mean,
|
||||
precipitation_sum = :precip,
|
||||
rain_sum = :rain,
|
||||
snowfall_sum = :snowfall,
|
||||
snow_depth_max = :snow_depth,
|
||||
windspeed_max = :wind,
|
||||
sunshine_hours = :sunshine,
|
||||
weathercode = :weathercode,
|
||||
fetched_at = NOW()
|
||||
"""),
|
||||
{
|
||||
"date": date_str,
|
||||
"temp_max": val("temperature_2m_max"),
|
||||
"temp_min": val("temperature_2m_min"),
|
||||
"temp_mean": val("temperature_2m_mean"),
|
||||
"precip": val("precipitation_sum"),
|
||||
"rain": val("rain_sum"),
|
||||
"snowfall": val("snowfall_sum"),
|
||||
"snow_depth": val("snow_depth_max"),
|
||||
"wind": val("windspeed_10m_max"),
|
||||
"sunshine": sunshine_h,
|
||||
"weathercode": val("weathercode"),
|
||||
}
|
||||
)
|
||||
records_upserted += 1
|
||||
|
||||
db.commit()
|
||||
|
||||
db.execute(
|
||||
text("""
|
||||
UPDATE sync_log
|
||||
SET completed_at = NOW(), status = 'success',
|
||||
records_fetched = :fetched, records_created = :created
|
||||
WHERE id = (
|
||||
SELECT id FROM sync_log
|
||||
WHERE source = 'open_meteo' AND status = 'running'
|
||||
ORDER BY started_at DESC LIMIT 1
|
||||
)
|
||||
"""),
|
||||
{"fetched": len(dates), "created": records_upserted}
|
||||
)
|
||||
db.commit()
|
||||
|
||||
logger.info(f"Weather sync completed: {records_upserted} days upserted")
|
||||
return {"records_fetched": len(dates), "records_upserted": records_upserted}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Weather sync failed: {e}")
|
||||
try:
|
||||
db.rollback()
|
||||
db.execute(
|
||||
text("""
|
||||
UPDATE sync_log
|
||||
SET completed_at = NOW(), status = 'failed', error_message = :error
|
||||
WHERE id = (
|
||||
SELECT id FROM sync_log
|
||||
WHERE source = 'open_meteo' AND status = 'running'
|
||||
ORDER BY started_at DESC LIMIT 1
|
||||
)
|
||||
"""),
|
||||
{"error": str(e)[:500]}
|
||||
)
|
||||
db.commit()
|
||||
except Exception as log_error:
|
||||
logger.error(f"Failed to update sync_log: {log_error}")
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def run_weather_sync(triggered_by: str = "scheduler"):
|
||||
"""Daily scheduled sync — last 7 days to catch ERA5 corrections."""
|
||||
from_date = date.today() - timedelta(days=7)
|
||||
to_date = date.today()
|
||||
await sync_weather_data(from_date, to_date, triggered_by=triggered_by)
|
||||
|
||||
|
||||
async def run_weather_backfill(
|
||||
from_date: date,
|
||||
to_date: date,
|
||||
triggered_by: str = "manual"
|
||||
) -> dict:
|
||||
"""Backfill in 6-month chunks so progress is logged incrementally."""
|
||||
logger.info(f"Starting weather backfill {from_date} to {to_date}")
|
||||
total_upserted = 0
|
||||
|
||||
chunk_start = from_date
|
||||
while chunk_start <= to_date:
|
||||
chunk_end = min(chunk_start + relativedelta(months=6) - timedelta(days=1), to_date)
|
||||
logger.info(f"Weather backfill chunk: {chunk_start} to {chunk_end}")
|
||||
result = await sync_weather_data(chunk_start, chunk_end, triggered_by=triggered_by)
|
||||
total_upserted += result.get("records_upserted", 0)
|
||||
chunk_start = chunk_end + timedelta(days=1)
|
||||
|
||||
logger.info(f"Weather backfill complete: {total_upserted} total days")
|
||||
return {"records_upserted": total_upserted}
|
||||
Loading…
Add table
Add a link
Reference in a new issue