From cabaff8d35e8557a51cbb73431c8015c30faddab Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Fri, 24 Jul 2026 21:26:02 +0000 Subject: [PATCH] =?UTF-8?q?Add=20rota=20backfill=20(60=20days)=20=E2=80=94?= =?UTF-8?q?=20separate=20from=20the=2025-month=20actuals=20backfill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runRollingSync only ever syncs wage_scheduled 14 days forward, so the AI insight's 28-day rota-vs-actual comparison had no historical rota to compare against for a freshly-scaffolded app. Adds a dedicated backfill action that pulls past schedules via the same throttled weekly-batch pattern as the existing actuals backfill. --- backend/src/lib/workforce.js | 38 +++++++++++++++++ backend/src/routes/sync.js | 52 +++++++++++++++++++++-- frontend/src/api.ts | 9 +++- frontend/src/pages/Settings.tsx | 75 ++++++++++++++++++++++++++++++--- 4 files changed, 164 insertions(+), 10 deletions(-) diff --git a/backend/src/lib/workforce.js b/backend/src/lib/workforce.js index 760104e..6a3f945 100644 --- a/backend/src/lib/workforce.js +++ b/backend/src/lib/workforce.js @@ -613,3 +613,41 @@ export async function runBackfill(onProgress, signal) { await new Promise(r => setTimeout(r, 250)) } } + +// Backfill wage_scheduled into the past. Unlike runRollingSync (which only ever syncs +// [today, today+14d] for schedules), rota-vs-actual comparisons need historical rota too — +// but rota loses relevance much faster than actuals, so this only reaches back 60 days +// (comfortably covers the AI insight's 28-day window) rather than runBackfill's 25 months. +export async function runScheduledBackfill(onProgress, signal, days = 60) { + const today = new Date() + const endDate = new Date(today) + endDate.setDate(endDate.getDate() - 1) + + const startDate = new Date(today) + startDate.setDate(startDate.getDate() - days) + + const totalDays = Math.max(1, Math.ceil((endDate - startDate) / 86_400_000)) + let processedDays = 0 + + let current = new Date(startDate) + while (current <= endDate) { + if (signal?.aborted) break + + const weekEnd = new Date(current) + weekEnd.setDate(weekEnd.getDate() + 6) + if (weekEnd > endDate) weekEnd.setTime(endDate.getTime()) + + const from = current.toISOString().slice(0, 10) + const to = weekEnd.toISOString().slice(0, 10) + + await syncScheduled(from, to) + + const daysInBatch = Math.ceil((weekEnd - current) / 86_400_000) + 1 + processedDays += daysInBatch + + onProgress?.({ processed: processedDays, total: totalDays, current: from }) + + current.setDate(current.getDate() + 7) + await new Promise(r => setTimeout(r, 250)) + } +} diff --git a/backend/src/routes/sync.js b/backend/src/routes/sync.js index 2fcccf6..5616c8a 100644 --- a/backend/src/routes/sync.js +++ b/backend/src/routes/sync.js @@ -1,18 +1,21 @@ import { requireAuth, requireCap } from '../auth.js' import { getConfig, setConfig } from '../db.js' -import { runRollingSync, runBackfill, compareEndpoints } from '../lib/workforce.js' +import { runRollingSync, runBackfill, runScheduledBackfill, compareEndpoints } from '../lib/workforce.js' import { fetchAllDepartments } from '../lib/workforce.js' let _backfillAbort = null +let _rotaBackfillAbort = null export async function syncRoutes(fastify) { fastify.addHook('preHandler', requireAuth) fastify.get('/api/sync/status', { preHandler: requireCap('view') }, async () => { return { - sync_last_at: await getConfig('sync_last_at'), - backfill_last_at: await getConfig('backfill_last_at'), - backfill_running: _backfillAbort !== null, + sync_last_at: await getConfig('sync_last_at'), + backfill_last_at: await getConfig('backfill_last_at'), + backfill_running: _backfillAbort !== null, + rota_backfill_last_at: await getConfig('rota_backfill_last_at'), + rota_backfill_running: _rotaBackfillAbort !== null, } }) @@ -66,6 +69,47 @@ export async function syncRoutes(fastify) { return { ok: true } }) + // Rota backfill — same SSE shape as /api/sync/backfill, but only reaches back 60 days + // (via runScheduledBackfill) since rota loses relevance much faster than actuals. + fastify.post('/api/sync/backfill/rota', { preHandler: requireCap('sync') }, async (request, reply) => { + if (_rotaBackfillAbort) { + _rotaBackfillAbort.abort() + _rotaBackfillAbort = null + } + + reply.raw.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + }) + + const ac = new AbortController() + _rotaBackfillAbort = ac + request.raw.on('close', () => ac.abort()) + + try { + await runScheduledBackfill(({ processed, total, current }) => { + reply.raw.write(`data: ${JSON.stringify({ processed, total, current })}\n\n`) + }, ac.signal) + + await setConfig('rota_backfill_last_at', new Date().toISOString()) + reply.raw.write(`data: ${JSON.stringify({ done: true })}\n\n`) + } catch (err) { + reply.raw.write(`data: ${JSON.stringify({ error: err.message })}\n\n`) + } finally { + _rotaBackfillAbort = null + reply.raw.end() + } + }) + + fastify.post('/api/sync/backfill/rota/cancel', { preHandler: requireCap('sync') }, async () => { + if (_rotaBackfillAbort) { + _rotaBackfillAbort.abort() + _rotaBackfillAbort = null + } + return { ok: true } + }) + // Compare cost totals from all available Workforce endpoints for a date range. // Useful for diagnosing figure discrepancies between shifts vs timesheets. // Usage: GET /wages/api/sync/compare?from=2025-06-01&to=2025-06-30 diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 3cbbb59..092bd8d 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -47,7 +47,10 @@ export function triggerSync(): Promise<{ ok: boolean; actual_rows: number; sched return request('/sync', { method: 'POST' }) } -export function getSyncStatus(): Promise<{ sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean }> { +export function getSyncStatus(): Promise<{ + sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean + rota_backfill_last_at: string | null; rota_backfill_running: boolean +}> { return request('/sync/status') } @@ -55,6 +58,10 @@ export function cancelBackfill(): Promise<{ ok: boolean }> { return request('/sync/backfill/cancel', { method: 'POST' }) } +export function cancelRotaBackfill(): Promise<{ ok: boolean }> { + return request('/sync/backfill/rota/cancel', { method: 'POST' }) +} + export function getDepartments(): Promise<{ departments: { id: string; name: string }[] }> { return request('/departments') } diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index bebe460..baa40e3 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -1,6 +1,6 @@ import { useState, useEffect } from 'react' import { RefreshCw, Download, X, CheckSquare, Square, Bot } from 'lucide-react' -import { getSettings, saveSettings, getDepartments, triggerSync, getSyncStatus, cancelBackfill, testAiInsightsConnection, generateInsight } from '../api' +import { getSettings, saveSettings, getDepartments, triggerSync, getSyncStatus, cancelBackfill, cancelRotaBackfill, testAiInsightsConnection, generateInsight } from '../api' import type { AppSetting, Department } from '../types' function fmtDate(iso: string | null): string { @@ -11,8 +11,12 @@ function fmtDate(iso: string | null): string { export default function SettingsPage() { const [settings, setSettings] = useState>({}) const [depts, setDepts] = useState([]) - const [syncStatus, setSyncStatus] = useState<{ sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean } | null>(null) - const [backfillProg, setBackfillProg] = useState<{ processed: number; total: number; current: string } | null>(null) + const [syncStatus, setSyncStatus] = useState<{ + sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean + rota_backfill_last_at: string | null; rota_backfill_running: boolean + } | null>(null) + const [backfillProg, setBackfillProg] = useState<{ processed: number; total: number; current: string } | null>(null) + const [rotaBackfillProg, setRotaBackfillProg] = useState<{ processed: number; total: number; current: string } | null>(null) const [syncing, setSyncing] = useState(false) const [loading, setLoading] = useState(true) const [fetchingDepts, setFetchingDepts] = useState(false) @@ -142,6 +146,43 @@ export default function SettingsPage() { setBackfillProg(null) } + const handleRotaBackfill = async () => { + if (!confirm('Backfill rota history? This will fetch the last 60 days of Workforce schedules.')) return + setRotaBackfillProg({ processed: 0, total: 1, current: '…' }) + setError(null) + + const es = new EventSource('/wages/api/sync/backfill/rota', { withCredentials: true }) + + const doPost = () => { + fetch('/wages/api/sync/backfill/rota', { + method: 'POST', + credentials: 'include', + }).catch(() => {}) + } + doPost() + + es.onmessage = (e) => { + const data = JSON.parse(e.data) + if (data.done) { + es.close() + setRotaBackfillProg(null) + setSyncStatus(s => s ? { ...s, rota_backfill_last_at: new Date().toISOString() } : s) + } else if (data.error) { + es.close() + setError(data.error) + setRotaBackfillProg(null) + } else { + setRotaBackfillProg(data) + } + } + es.onerror = () => { es.close(); setRotaBackfillProg(null) } + } + + const handleCancelRotaBackfill = async () => { + await cancelRotaBackfill() + setRotaBackfillProg(null) + } + const handleTestConnection = async () => { setTestStatus('testing'); setTestMessage('') try { @@ -374,7 +415,7 @@ export default function SettingsPage() { {syncing ? 'Syncing…' : 'Sync Now (35 days)'} - @@ -383,6 +424,15 @@ export default function SettingsPage() { Cancel )} + + {rotaBackfillProg && ( + + )} {backfillProg && ( @@ -397,13 +447,28 @@ export default function SettingsPage() { )} + {rotaBackfillProg && ( +
+
+ Fetching rota {rotaBackfillProg.current}… + {rotaBackfillProg.processed} / {rotaBackfillProg.total} days +
+
+
+
+
+ )} +
Last sync: {fmtDate(syncStatus?.sync_last_at ?? null)}
Last backfill: {fmtDate(syncStatus?.backfill_last_at ?? null)}
+
Last rota backfill: {fmtDate(syncStatus?.rota_backfill_last_at ?? null)}

Sync Now pulls the last 35 days of timesheets + next 14 days of schedules. Auto-sync runs every hour. - Deep Backfill fetches the full 13-month history at 250ms per week to avoid rate limits. + Deep Backfill fetches the full 25-month actuals history at 250ms per week to avoid rate limits. + Backfill Rota fetches the last 60 days of published/draft schedules — needed for the rota-vs-actual + comparison in AI Insights, since normal sync only ever looks 14 days ahead for rota.