Add rota backfill (60 days) — separate from the 25-month actuals backfill
runRollingSync only ever syncs wage_scheduled 14 days forward, so the AI insight's 28-day rota-vs-actual comparison had no historical rota to compare against for a freshly-scaffolded app. Adds a dedicated backfill action that pulls past schedules via the same throttled weekly-batch pattern as the existing actuals backfill.
This commit is contained in:
parent
95da5ea237
commit
cabaff8d35
4 changed files with 164 additions and 10 deletions
|
|
@ -613,3 +613,41 @@ export async function runBackfill(onProgress, signal) {
|
|||
await new Promise(r => setTimeout(r, 250))
|
||||
}
|
||||
}
|
||||
|
||||
// Backfill wage_scheduled into the past. Unlike runRollingSync (which only ever syncs
|
||||
// [today, today+14d] for schedules), rota-vs-actual comparisons need historical rota too —
|
||||
// but rota loses relevance much faster than actuals, so this only reaches back 60 days
|
||||
// (comfortably covers the AI insight's 28-day window) rather than runBackfill's 25 months.
|
||||
export async function runScheduledBackfill(onProgress, signal, days = 60) {
|
||||
const today = new Date()
|
||||
const endDate = new Date(today)
|
||||
endDate.setDate(endDate.getDate() - 1)
|
||||
|
||||
const startDate = new Date(today)
|
||||
startDate.setDate(startDate.getDate() - days)
|
||||
|
||||
const totalDays = Math.max(1, Math.ceil((endDate - startDate) / 86_400_000))
|
||||
let processedDays = 0
|
||||
|
||||
let current = new Date(startDate)
|
||||
while (current <= endDate) {
|
||||
if (signal?.aborted) break
|
||||
|
||||
const weekEnd = new Date(current)
|
||||
weekEnd.setDate(weekEnd.getDate() + 6)
|
||||
if (weekEnd > endDate) weekEnd.setTime(endDate.getTime())
|
||||
|
||||
const from = current.toISOString().slice(0, 10)
|
||||
const to = weekEnd.toISOString().slice(0, 10)
|
||||
|
||||
await syncScheduled(from, to)
|
||||
|
||||
const daysInBatch = Math.ceil((weekEnd - current) / 86_400_000) + 1
|
||||
processedDays += daysInBatch
|
||||
|
||||
onProgress?.({ processed: processedDays, total: totalDays, current: from })
|
||||
|
||||
current.setDate(current.getDate() + 7)
|
||||
await new Promise(r => setTimeout(r, 250))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
import { requireAuth, requireCap } from '../auth.js'
|
||||
import { getConfig, setConfig } from '../db.js'
|
||||
import { runRollingSync, runBackfill, compareEndpoints } from '../lib/workforce.js'
|
||||
import { runRollingSync, runBackfill, runScheduledBackfill, compareEndpoints } from '../lib/workforce.js'
|
||||
import { fetchAllDepartments } from '../lib/workforce.js'
|
||||
|
||||
let _backfillAbort = null
|
||||
let _rotaBackfillAbort = null
|
||||
|
||||
export async function syncRoutes(fastify) {
|
||||
fastify.addHook('preHandler', requireAuth)
|
||||
|
||||
fastify.get('/api/sync/status', { preHandler: requireCap('view') }, async () => {
|
||||
return {
|
||||
sync_last_at: await getConfig('sync_last_at'),
|
||||
backfill_last_at: await getConfig('backfill_last_at'),
|
||||
backfill_running: _backfillAbort !== null,
|
||||
sync_last_at: await getConfig('sync_last_at'),
|
||||
backfill_last_at: await getConfig('backfill_last_at'),
|
||||
backfill_running: _backfillAbort !== null,
|
||||
rota_backfill_last_at: await getConfig('rota_backfill_last_at'),
|
||||
rota_backfill_running: _rotaBackfillAbort !== null,
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -66,6 +69,47 @@ export async function syncRoutes(fastify) {
|
|||
return { ok: true }
|
||||
})
|
||||
|
||||
// Rota backfill — same SSE shape as /api/sync/backfill, but only reaches back 60 days
|
||||
// (via runScheduledBackfill) since rota loses relevance much faster than actuals.
|
||||
fastify.post('/api/sync/backfill/rota', { preHandler: requireCap('sync') }, async (request, reply) => {
|
||||
if (_rotaBackfillAbort) {
|
||||
_rotaBackfillAbort.abort()
|
||||
_rotaBackfillAbort = null
|
||||
}
|
||||
|
||||
reply.raw.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
})
|
||||
|
||||
const ac = new AbortController()
|
||||
_rotaBackfillAbort = ac
|
||||
request.raw.on('close', () => ac.abort())
|
||||
|
||||
try {
|
||||
await runScheduledBackfill(({ processed, total, current }) => {
|
||||
reply.raw.write(`data: ${JSON.stringify({ processed, total, current })}\n\n`)
|
||||
}, ac.signal)
|
||||
|
||||
await setConfig('rota_backfill_last_at', new Date().toISOString())
|
||||
reply.raw.write(`data: ${JSON.stringify({ done: true })}\n\n`)
|
||||
} catch (err) {
|
||||
reply.raw.write(`data: ${JSON.stringify({ error: err.message })}\n\n`)
|
||||
} finally {
|
||||
_rotaBackfillAbort = null
|
||||
reply.raw.end()
|
||||
}
|
||||
})
|
||||
|
||||
fastify.post('/api/sync/backfill/rota/cancel', { preHandler: requireCap('sync') }, async () => {
|
||||
if (_rotaBackfillAbort) {
|
||||
_rotaBackfillAbort.abort()
|
||||
_rotaBackfillAbort = null
|
||||
}
|
||||
return { ok: true }
|
||||
})
|
||||
|
||||
// Compare cost totals from all available Workforce endpoints for a date range.
|
||||
// Useful for diagnosing figure discrepancies between shifts vs timesheets.
|
||||
// Usage: GET /wages/api/sync/compare?from=2025-06-01&to=2025-06-30
|
||||
|
|
|
|||
|
|
@ -47,7 +47,10 @@ export function triggerSync(): Promise<{ ok: boolean; actual_rows: number; sched
|
|||
return request('/sync', { method: 'POST' })
|
||||
}
|
||||
|
||||
export function getSyncStatus(): Promise<{ sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean }> {
|
||||
export function getSyncStatus(): Promise<{
|
||||
sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean
|
||||
rota_backfill_last_at: string | null; rota_backfill_running: boolean
|
||||
}> {
|
||||
return request('/sync/status')
|
||||
}
|
||||
|
||||
|
|
@ -55,6 +58,10 @@ export function cancelBackfill(): Promise<{ ok: boolean }> {
|
|||
return request('/sync/backfill/cancel', { method: 'POST' })
|
||||
}
|
||||
|
||||
export function cancelRotaBackfill(): Promise<{ ok: boolean }> {
|
||||
return request('/sync/backfill/rota/cancel', { method: 'POST' })
|
||||
}
|
||||
|
||||
export function getDepartments(): Promise<{ departments: { id: string; name: string }[] }> {
|
||||
return request('/departments')
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { useState, useEffect } from 'react'
|
||||
import { RefreshCw, Download, X, CheckSquare, Square, Bot } from 'lucide-react'
|
||||
import { getSettings, saveSettings, getDepartments, triggerSync, getSyncStatus, cancelBackfill, testAiInsightsConnection, generateInsight } from '../api'
|
||||
import { getSettings, saveSettings, getDepartments, triggerSync, getSyncStatus, cancelBackfill, cancelRotaBackfill, testAiInsightsConnection, generateInsight } from '../api'
|
||||
import type { AppSetting, Department } from '../types'
|
||||
|
||||
function fmtDate(iso: string | null): string {
|
||||
|
|
@ -11,8 +11,12 @@ function fmtDate(iso: string | null): string {
|
|||
export default function SettingsPage() {
|
||||
const [settings, setSettings] = useState<Record<string, string>>({})
|
||||
const [depts, setDepts] = useState<Department[]>([])
|
||||
const [syncStatus, setSyncStatus] = useState<{ sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean } | null>(null)
|
||||
const [backfillProg, setBackfillProg] = useState<{ processed: number; total: number; current: string } | null>(null)
|
||||
const [syncStatus, setSyncStatus] = useState<{
|
||||
sync_last_at: string | null; backfill_last_at: string | null; backfill_running: boolean
|
||||
rota_backfill_last_at: string | null; rota_backfill_running: boolean
|
||||
} | null>(null)
|
||||
const [backfillProg, setBackfillProg] = useState<{ processed: number; total: number; current: string } | null>(null)
|
||||
const [rotaBackfillProg, setRotaBackfillProg] = useState<{ processed: number; total: number; current: string } | null>(null)
|
||||
const [syncing, setSyncing] = useState(false)
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [fetchingDepts, setFetchingDepts] = useState(false)
|
||||
|
|
@ -142,6 +146,43 @@ export default function SettingsPage() {
|
|||
setBackfillProg(null)
|
||||
}
|
||||
|
||||
const handleRotaBackfill = async () => {
|
||||
if (!confirm('Backfill rota history? This will fetch the last 60 days of Workforce schedules.')) return
|
||||
setRotaBackfillProg({ processed: 0, total: 1, current: '…' })
|
||||
setError(null)
|
||||
|
||||
const es = new EventSource('/wages/api/sync/backfill/rota', { withCredentials: true })
|
||||
|
||||
const doPost = () => {
|
||||
fetch('/wages/api/sync/backfill/rota', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
}).catch(() => {})
|
||||
}
|
||||
doPost()
|
||||
|
||||
es.onmessage = (e) => {
|
||||
const data = JSON.parse(e.data)
|
||||
if (data.done) {
|
||||
es.close()
|
||||
setRotaBackfillProg(null)
|
||||
setSyncStatus(s => s ? { ...s, rota_backfill_last_at: new Date().toISOString() } : s)
|
||||
} else if (data.error) {
|
||||
es.close()
|
||||
setError(data.error)
|
||||
setRotaBackfillProg(null)
|
||||
} else {
|
||||
setRotaBackfillProg(data)
|
||||
}
|
||||
}
|
||||
es.onerror = () => { es.close(); setRotaBackfillProg(null) }
|
||||
}
|
||||
|
||||
const handleCancelRotaBackfill = async () => {
|
||||
await cancelRotaBackfill()
|
||||
setRotaBackfillProg(null)
|
||||
}
|
||||
|
||||
const handleTestConnection = async () => {
|
||||
setTestStatus('testing'); setTestMessage('')
|
||||
try {
|
||||
|
|
@ -374,7 +415,7 @@ export default function SettingsPage() {
|
|||
<RefreshCw size={14} strokeWidth={1.75} />
|
||||
{syncing ? 'Syncing…' : 'Sync Now (35 days)'}
|
||||
</button>
|
||||
<button className="btn btn-secondary" onClick={handleBackfill} disabled={syncing || backfillProg != null}>
|
||||
<button className="btn btn-secondary" onClick={handleBackfill} disabled={syncing || backfillProg != null || rotaBackfillProg != null}>
|
||||
<Download size={14} strokeWidth={1.75} />
|
||||
Deep Backfill (25 months)
|
||||
</button>
|
||||
|
|
@ -383,6 +424,15 @@ export default function SettingsPage() {
|
|||
<X size={14} strokeWidth={1.75} /> Cancel
|
||||
</button>
|
||||
)}
|
||||
<button className="btn btn-secondary" onClick={handleRotaBackfill} disabled={syncing || backfillProg != null || rotaBackfillProg != null}>
|
||||
<Download size={14} strokeWidth={1.75} />
|
||||
Backfill Rota (60 days)
|
||||
</button>
|
||||
{rotaBackfillProg && (
|
||||
<button className="btn btn-secondary" onClick={handleCancelRotaBackfill}>
|
||||
<X size={14} strokeWidth={1.75} /> Cancel
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{backfillProg && (
|
||||
|
|
@ -397,13 +447,28 @@ export default function SettingsPage() {
|
|||
</div>
|
||||
)}
|
||||
|
||||
{rotaBackfillProg && (
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', fontSize: 12, marginBottom: 4 }}>
|
||||
<span>Fetching rota {rotaBackfillProg.current}…</span>
|
||||
<span>{rotaBackfillProg.processed} / {rotaBackfillProg.total} days</span>
|
||||
</div>
|
||||
<div style={{ height: 6, background: 'var(--border)', borderRadius: 3 }}>
|
||||
<div style={{ height: '100%', width: `${Math.min(100, (rotaBackfillProg.processed / rotaBackfillProg.total) * 100)}%`, background: 'var(--app-primary)', borderRadius: 3, transition: 'width 0.3s' }} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={{ fontSize: 13, color: 'var(--text-muted)', display: 'grid', gap: 4 }}>
|
||||
<div>Last sync: <strong>{fmtDate(syncStatus?.sync_last_at ?? null)}</strong></div>
|
||||
<div>Last backfill: <strong>{fmtDate(syncStatus?.backfill_last_at ?? null)}</strong></div>
|
||||
<div>Last rota backfill: <strong>{fmtDate(syncStatus?.rota_backfill_last_at ?? null)}</strong></div>
|
||||
</div>
|
||||
<p style={{ marginTop: 8, fontSize: 12, color: 'var(--text-muted)' }}>
|
||||
Sync Now pulls the last 35 days of timesheets + next 14 days of schedules. Auto-sync runs every hour.
|
||||
Deep Backfill fetches the full 13-month history at 250ms per week to avoid rate limits.
|
||||
Deep Backfill fetches the full 25-month actuals history at 250ms per week to avoid rate limits.
|
||||
Backfill Rota fetches the last 60 days of published/draft schedules — needed for the rota-vs-actual
|
||||
comparison in AI Insights, since normal sync only ever looks 14 days ahead for rota.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue