wages/backend/src/jobs/ai-insights.js

import Anthropic from '@anthropic-ai/sdk'
import { pool, getConfig } from '../db.js'
import { getAnthropicApiKey } from '../lib/central-settings.js'
import { fcFetch } from '../lib/forecasting-client.js'
import { forecastDayCost } from '../lib/forecast.js'

export const DEFAULT_MODEL = 'claude-haiku-4-5-20251001'
export const DEFAULT_DAILY_TOKEN_BUDGET = 5000
const MAX_OUTPUT_TOKENS = 600
export const MANUAL_RATE_LIMIT_MINUTES = 5

function toISODate(d) {
  return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
}

function fmtDateUK(isoDate) {
  const [y, m, d] = String(isoDate).split('-')
  return `${d}/${m}/${y}`
}

function fmtMoney(n) {
  return n == null ? '-' : `£${Number(n).toLocaleString('en-GB', { maximumFractionDigits: 0 })}`
}

function fmtPct(n) {
  return n == null ? '-' : `${(n * 100).toFixed(0)}%`
}

async function getCostCol() {
  const showOncosts = (await getConfig('show_oncosts')) !== 'false'
  return showOncosts ? 'total_cost' : 'base_cost'
}

export async function getAiInsightsConfig() {
  const [enabled, model, scheduleTime, dailyTokenBudget] = await Promise.all([
    getConfig('ai_insights_enabled'),
    getConfig('ai_insights_model'),
    getConfig('ai_insights_schedule_time'),
    getConfig('ai_insights_daily_token_budget'),
  ])
  return {
    enabled: enabled === 'true',
    model: model || DEFAULT_MODEL,
    scheduleTime: scheduleTime || '07:15',
    dailyTokenBudget: parseInt(dailyTokenBudget, 10) || DEFAULT_DAILY_TOKEN_BUDGET,
  }
}

// Last few generated insights, most recent first — fed back into the prompt so the
// model can reference what it already said instead of repeating itself verbatim.
export async function getRecentInsights(limit = 3) {
  const res = await pool.query(
    `SELECT generated_at, content FROM ai_insights ORDER BY generated_at DESC LIMIT $1`,
    [limit]
  )
  return res.rows
}

export async function checkDailyBudget(budgetTokens) {
  const res = await pool.query(
    `SELECT COALESCE(SUM(input_tokens + output_tokens), 0) AS total
     FROM ai_insights WHERE generated_at >= CURRENT_DATE`
  )
  const usedToday = parseInt(res.rows[0].total, 10)
  return { withinBudget: usedToday < budgetTokens, usedToday }
}

// Month-to-date total wages cost (company-wide) vs the total month budget.
// No per-department budget split — that split is a rough estimate and skews comparisons.
export async function gatherMonthProgressData() {
  const costCol = await getCostCol()
  const today = new Date()
  const monthStart = `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}-01`
  const yesterdayStr = toISODate(new Date(today.getTime() - 86_400_000))
  const daysInMonth = new Date(today.getFullYear(), today.getMonth() + 1, 0).getDate()
  const daysElapsed = Math.max(0, today.getDate() - 1)

  const [actualRes, budgetRes] = await Promise.all([
    pool.query(
      `SELECT COALESCE(SUM(${costCol}), 0) AS mtd_cost
       FROM wage_actuals WHERE date >= $1 AND date <= $2`,
      [monthStart, yesterdayStr]
    ),
    pool.query(`SELECT budget_amount FROM wage_budgets WHERE month = $1`, [monthStart]),
  ])

  const mtdCost = parseFloat(actualRes.rows[0].mtd_cost)
  const budgetAmount = budgetRes.rows[0] ? parseFloat(budgetRes.rows[0].budget_amount) : null

  return {
    monthStart,
    monthLabel: monthStart.slice(0, 7),
    yesterdayStr,
    daysElapsed,
    daysInMonth,
    pctMonthElapsed: daysElapsed / daysInMonth,
    mtdCost,
    budgetAmount,
    pctBudgetUsed: budgetAmount ? mtdCost / budgetAmount : null,
  }
}

// Full previous month and full same-month-last-year totals (complete datasets, not
// sliced to a matching to-date range) — the model draws its own pace comparison.
export async function gatherPriorPeriodData() {
  const costCol = await getCostCol()
  const today = new Date()

  const prevMonthStart = toISODate(new Date(today.getFullYear(), today.getMonth() - 1, 1))
  const prevMonthEnd = toISODate(new Date(today.getFullYear(), today.getMonth(), 0))
  const lastYearStart = toISODate(new Date(today.getFullYear() - 1, today.getMonth(), 1))
  const lastYearEnd = toISODate(new Date(today.getFullYear() - 1, today.getMonth() + 1, 0))

  const shapeDepts = rows => rows.map(r => ({
    department_id: r.department_id,
    department_name: r.department_name,
    cost: parseFloat(r.cost),
  }))

  const deptQuery = (from, to) => pool.query(
    `SELECT department_id, department_name, SUM(${costCol}) AS cost
     FROM wage_actuals WHERE date >= $1 AND date <= $2
     GROUP BY department_id, department_name ORDER BY SUM(${costCol}) DESC`,
    [from, to]
  )

  const [prevMonthRes, lastYearRes] = await Promise.all([
    deptQuery(prevMonthStart, prevMonthEnd),
    deptQuery(lastYearStart, lastYearEnd),
  ])

  const prevMonthDepts = shapeDepts(prevMonthRes.rows)
  const lastYearDepts = shapeDepts(lastYearRes.rows)

  return {
    prevMonthLabel: prevMonthStart.slice(0, 7),
    prevMonthTotal: prevMonthDepts.reduce((s, d) => s + d.cost, 0),
    prevMonthDepts,
    lastYearLabel: lastYearStart.slice(0, 7),
    lastYearTotal: lastYearDepts.reduce((s, d) => s + d.cost, 0),
    lastYearDepts,
  }
}

// Department-level variance between scheduled (rota) and actual cost, over the trailing
// window. Rota rows for past dates aren't deleted once synced, so this covers real history.
//
// Always compares on BASE cost for both sides, regardless of the show_oncosts display
// setting: Workforce's schedules API never actually supplies employer NI oncosts (confirmed —
// published_total_cost is identical to published_base_cost on every synced row), so comparing
// oncost-inclusive actuals against rota would inflate every variance by ~14-20% for reasons
// that have nothing to do with real overspend. Matches the same caveat already surfaced on the
// Monthly page's rota-based forecast footnote.
export async function gatherRotaVsActualData(days = 28) {
  const actualCostCol = 'base_cost'
  const schedCostExpr = '(published_base_cost + unpublished_base_cost)'

  const today = new Date()
  const fromStr = toISODate(new Date(today.getTime() - days * 86_400_000))
  const toStr = toISODate(new Date(today.getTime() - 86_400_000))

  const [actualRes, schedRes] = await Promise.all([
    pool.query(
      `SELECT date, department_id, department_name, ${actualCostCol} AS cost
       FROM wage_actuals WHERE date >= $1 AND date <= $2`,
      [fromStr, toStr]
    ),
    pool.query(
      `SELECT date, department_id, ${schedCostExpr} AS cost
       FROM wage_scheduled WHERE date >= $1 AND date <= $2`,
      [fromStr, toStr]
    ),
  ])

  const totalDaysInWindow = Math.round((new Date(toStr + 'T00:00:00') - new Date(fromStr + 'T00:00:00')) / 86_400_000) + 1

  const schedMap = {}
  for (const r of schedRes.rows) {
    schedMap[`${r.date.toISOString().slice(0, 10)}:${r.department_id}`] = parseFloat(r.cost)
  }

  // actualTotal = full-window actual cost (context only). actualOnRotaDays = actual cost
  // restricted to the SAME days rota data exists for — this is what variance is computed
  // from, so a department with sparse rota history doesn't get a wildly inflated "overspend"
  // that's really just missing rota rows, not real cost variance.
  const byDept = {}
  for (const r of actualRes.rows) {
    const dateStr = r.date.toISOString().slice(0, 10)
    const dep = r.department_id
    byDept[dep] ??= { department_id: dep, department_name: r.department_name, actualTotal: 0, actualOnRotaDays: 0, schedTotal: 0, daysWithRota: 0 }
    const actualCost = parseFloat(r.cost)
    const schedCost = schedMap[`${dateStr}:${dep}`]
    byDept[dep].actualTotal += actualCost
    if (schedCost != null) {
      byDept[dep].schedTotal += schedCost
      byDept[dep].actualOnRotaDays += actualCost
      byDept[dep].daysWithRota++
    }
  }

  const depts = Object.values(byDept)
    .map(d => ({
      ...d,
      variance: d.actualOnRotaDays - d.schedTotal,
      variancePct: d.schedTotal > 0 ? (d.actualOnRotaDays - d.schedTotal) / d.schedTotal : null,
    }))
    .sort((a, b) => Math.abs(b.variance) - Math.abs(a.variance))

  return { fromStr, toStr, totalDaysInWindow, depts }
}

// Forward-looking projection — reuses the app's existing tiered forecastDayCost() logic
// and the forecast_method setting already used by the Monthly/Weekly pages, so this never
// disagrees with what those pages show. Not related to forecasting app's revenue model.
export async function gatherForecastData(monthProgress) {
  const showOncosts = (await getConfig('show_oncosts')) !== 'false'
  const forecastMethodRaw = await getConfig('forecast_method')
  const forecastMethod = forecastMethodRaw === 'rota' ? 'rota' : 'repeat'

  const today = new Date()
  const todayStr = toISODate(today)
  const monthEndDate = new Date(today.getFullYear(), today.getMonth() + 1, 0)
  const monthEndStr = toISODate(monthEndDate)
  const actualsFrom = toISODate(new Date(today.getTime() - 42 * 86_400_000))

  const [actualRes, schedRes] = await Promise.all([
    pool.query(
      `SELECT date, department_id, department_name, ${showOncosts ? 'total_cost' : 'base_cost'} AS cost
       FROM wage_actuals WHERE date >= $1 AND date < $2`,
      [actualsFrom, todayStr]
    ),
    pool.query(
      `SELECT date, department_id, department_name,
              published_base_cost, published_total_cost, published_shift_count,
              unpublished_base_cost, unpublished_total_cost, unpublished_shift_count
       FROM wage_scheduled WHERE date >= $1 AND date <= $2`,
      [todayStr, monthEndStr]
    ),
  ])

  const deptNames = {}
  const actualByDept = {}
  for (const r of actualRes.rows) {
    const dep = r.department_id
    deptNames[dep] = r.department_name
    actualByDept[dep] ??= {}
    actualByDept[dep][r.date.toISOString().slice(0, 10)] = { cost: parseFloat(r.cost) }
  }

  const schedByDept = {}
  for (const r of schedRes.rows) {
    const dep = r.department_id
    deptNames[dep] = r.department_name
    schedByDept[dep] ??= {}
    schedByDept[dep][r.date.toISOString().slice(0, 10)] = {
      published_cost: parseFloat(showOncosts ? r.published_total_cost : r.published_base_cost),
      unpublished_cost: parseFloat(showOncosts ? r.unpublished_total_cost : r.unpublished_base_cost),
      published_shift_count: r.published_shift_count,
      unpublished_shift_count: r.unpublished_shift_count,
    }
  }

  const remainingDays = []
  for (let d = new Date(today); d <= monthEndDate; d.setDate(d.getDate() + 1)) {
    remainingDays.push(toISODate(d))
  }

  let remainingTotal = 0
  const byDept = []
  for (const dep of Object.keys(deptNames)) {
    const schedDep = forecastMethod === 'rota' ? schedByDept[dep] : undefined
    let depRemaining = 0
    for (const dateStr of remainingDays) {
      depRemaining += forecastDayCost(dateStr, actualByDept[dep], schedDep, false).cost
    }
    remainingTotal += depRemaining
    byDept.push({ department_id: dep, department_name: deptNames[dep], remaining: depRemaining })
  }

  const projectedTotal = (monthProgress?.mtdCost ?? 0) + remainingTotal
  return {
    forecastMethod,
    remainingDaysCount: remainingDays.length,
    remainingTotal,
    projectedTotal,
    byDept: byDept.sort((a, b) => b.remaining - a.remaining),
  }
}

// Individual shifts/wages worth a look — top-cost employees over the trailing week.
export async function gatherEmployeeAnomalies(days = 7) {
  const costCol = await getCostCol()
  const today = new Date()
  const fromStr = toISODate(new Date(today.getTime() - days * 86_400_000))
  const toStr = toISODate(new Date(today.getTime() - 86_400_000))

  // wage_actuals_detail only stores department_id (a raw Workforce code, e.g. "972312"),
  // not a human-readable name — resolve it from wage_actuals, which has both, so the
  // prompt (and the model) never has to guess which department a code refers to.
  const [res, deptNamesRes] = await Promise.all([
    pool.query(
      `SELECT employee_id, employee_name, department_id,
              SUM(${costCol}) AS cost, SUM(shift_count)::int AS shift_count
       FROM wage_actuals_detail
       WHERE date >= $1 AND date <= $2
       GROUP BY employee_id, employee_name, department_id
       ORDER BY SUM(${costCol}) DESC
       LIMIT 15`,
      [fromStr, toStr]
    ),
    pool.query(`SELECT DISTINCT department_id, department_name FROM wage_actuals`),
  ])

  const deptNames = Object.fromEntries(deptNamesRes.rows.map(r => [r.department_id, r.department_name]))

  return {
    fromStr,
    toStr,
    employees: res.rows.map(r => ({
      employee_id: r.employee_id,
      employee_name: r.employee_name,
      department_id: r.department_id,
      department_name: deptNames[r.department_id] || r.department_id,
      cost: parseFloat(r.cost),
      shift_count: r.shift_count,
    })),
  }
}

// Wage cost as a % of net sales, month-to-date — reuses the same cross-app call net-sales.js
// already makes to the forecasting app's public revenue API.
export async function gatherRevenueCorrelation(monthProgress) {
  try {
    const days = Math.max(1, monthProgress.daysElapsed)
    const data = await fcFetch(`/forecast/revenue?start_date=${monthProgress.monthStart}&days=${days}&type=all&dow_align=true`)
    const rows = data?.data ?? []
    const totalRevenue = rows.reduce((s, d) => s + parseFloat(d.total?.forecast ?? d.total?.otb ?? 0), 0)
    return {
      totalRevenue,
      wagePct: totalRevenue > 0 ? (monthProgress.mtdCost / totalRevenue) * 100 : null,
      daysCovered: rows.length,
    }
  } catch (e) {
    return { error: e.message }
  }
}

export function buildPrompt(monthProgress, priorPeriod, rotaVsActual, forecast, anomalies, revenueCorrelation, recentInsights = []) {
  const systemMsg =
    "You are an AI assistant for a wage cost controller in a UK hospitality business. Analyze the data below and " +
    "provide a concise daily briefing (3-5 bullet points). Focus on: how this month's wage cost is tracking " +
    "against budget, the month-end forecast, notable variance between rota and actual cost by department and " +
    "what might be causing it, individual pay anomalies worth a look, and wage cost as a percentage of revenue. " +
    "Use UK date format (DD/MM/YYYY) and GBP (£) for all monetary values. Be specific with department/employee " +
    "names and numbers. Keep it actionable — no fluff or generic advice.\n\n" +
    "A 'Recent Previous Insights' section may be included below — compare against them explicitly: call out " +
    "what's changed, what's resolved, and what's still an open issue. Don't just repeat the same points verbatim."

  const lines = []

  if (recentInsights.length) {
    lines.push('## Recent Previous Insights (most recent first — reference these, do not just repeat them)')
    for (const r of recentInsights) {
      const when = new Date(r.generated_at).toLocaleString('en-GB', { day: '2-digit', month: '2-digit', year: 'numeric', hour: '2-digit', minute: '2-digit' })
      lines.push(`${when}:`)
      lines.push(r.content)
      lines.push('')
    }
  }

  lines.push(`## Month to Date (${monthProgress.monthLabel})`)
  lines.push(
    `Day ${monthProgress.daysElapsed} of ${monthProgress.daysInMonth} (${fmtPct(monthProgress.pctMonthElapsed)} of month elapsed)`
  )
  lines.push(
    `MTD cost: ${fmtMoney(monthProgress.mtdCost)} | Month budget: ${fmtMoney(monthProgress.budgetAmount)}` +
    (monthProgress.pctBudgetUsed != null ? ` (${fmtPct(monthProgress.pctBudgetUsed)} used)` : '')
  )
  lines.push('')

  lines.push('## Prior Period Comparison (full-month totals, for context — not a budget split)')
  lines.push(`Previous month (${priorPeriod.prevMonthLabel}) total: ${fmtMoney(priorPeriod.prevMonthTotal)}`)
  for (const d of priorPeriod.prevMonthDepts) lines.push(`  ${d.department_name}: ${fmtMoney(d.cost)}`)
  lines.push(`Same month last year (${priorPeriod.lastYearLabel}) total: ${fmtMoney(priorPeriod.lastYearTotal)}`)
  for (const d of priorPeriod.lastYearDepts) lines.push(`  ${d.department_name}: ${fmtMoney(d.cost)}`)
  lines.push('')

  lines.push(`## Rota vs Actual Variance (${fmtDateUK(rotaVsActual.fromStr)} - ${fmtDateUK(rotaVsActual.toStr)}, ${rotaVsActual.totalDaysInWindow} days)`)
  lines.push(
    "Figures below are BASE PAY only (excluding employer NI) on both sides — Workforce's rota/schedules " +
    "API doesn't supply oncosts, so this is the only basis that's genuinely comparable; do not describe " +
    "this variance using total-cost figures from other sections. Variance is computed only over days that " +
    "have rota data (\"days w/ rota\" below) — if that's well below the total window, treat the variance " +
    "as partial/uncertain due to missing rota history, not a confirmed overspend, and say so explicitly."
  )
  lines.push('Department | Actual base pay (days w/ rota) | Rota base pay | Variance | Variance % | Days w/ rota | Actual base pay (full window)')
  for (const d of rotaVsActual.depts) {
    lines.push(
      `${d.department_name} | ${fmtMoney(d.actualOnRotaDays)} | ${fmtMoney(d.schedTotal)} | ` +
      `${fmtMoney(d.variance)} | ${fmtPct(d.variancePct)} | ${d.daysWithRota}/${rotaVsActual.totalDaysInWindow} | ${fmtMoney(d.actualTotal)}`
    )
  }
  lines.push('')

  lines.push(`## Forecast (method: ${forecast.forecastMethod}, ${forecast.remainingDaysCount} days remaining)`)
  if (forecast.forecastMethod === 'rota') {
    lines.push(
      "Note: remaining days sourced from rota exclude employer NI oncosts (Workforce's schedules API " +
      "doesn't supply them), so this projection may modestly understate the true month-end total — same " +
      "known limitation shown on the Monthly page."
    )
  }
  lines.push(
    `Projected month-end total: ${fmtMoney(forecast.projectedTotal)} vs budget ${fmtMoney(monthProgress.budgetAmount)}` +
    (monthProgress.budgetAmount != null
      ? ` (${forecast.projectedTotal > monthProgress.budgetAmount ? 'over' : 'under'} by ${fmtMoney(Math.abs(forecast.projectedTotal - monthProgress.budgetAmount))})`
      : '')
  )
  for (const d of forecast.byDept) lines.push(`  ${d.department_name} remaining: ${fmtMoney(d.remaining)}`)
  lines.push('')

  lines.push(`## Notable Individual Shifts/Wages (${fmtDateUK(anomalies.fromStr)} - ${fmtDateUK(anomalies.toStr)})`)
  lines.push('Employee | Department | Cost | Shifts')
  for (const e of anomalies.employees) {
    lines.push(`${e.employee_name} | ${e.department_name} | ${fmtMoney(e.cost)} | ${e.shift_count}`)
  }
  lines.push('')

  lines.push('## Wage Cost vs Revenue (month to date)')
  if (revenueCorrelation.error) {
    lines.push(`Unavailable: ${revenueCorrelation.error}`)
  } else {
    lines.push(
      `Revenue: ${fmtMoney(revenueCorrelation.totalRevenue)} | Wage cost: ${fmtMoney(monthProgress.mtdCost)} | ` +
      `Wage %: ${revenueCorrelation.wagePct != null ? revenueCorrelation.wagePct.toFixed(1) + '%' : '-'}`
    )
  }

  return { systemMsg, userMsg: lines.join('\n') }
}

export async function callLlm(apiKey, systemMsg, userMsg, model) {
  const client = new Anthropic({ apiKey })
  const response = await client.messages.create({
    model,
    max_tokens: MAX_OUTPUT_TOKENS,
    temperature: 0.2,
    system: systemMsg,
    messages: [{ role: 'user', content: userMsg }],
  })
  const content = response.content?.[0]?.text ?? ''
  return {
    content,
    input_tokens: response.usage.input_tokens,
    output_tokens: response.usage.output_tokens,
    model,
  }
}

export async function saveInsight({ content, model, input_tokens, output_tokens, data_snapshot, triggered_by }) {
  await pool.query(
    `INSERT INTO ai_insights (content, model, input_tokens, output_tokens, data_snapshot, triggered_by)
     VALUES ($1, $2, $3, $4, $5::jsonb, $6)`,
    [content, model, input_tokens, output_tokens, JSON.stringify(data_snapshot), triggered_by]
  )
}

export async function cleanupOldInsights(keepDays = 90) {
  await pool.query(`DELETE FROM ai_insights WHERE generated_at < NOW() - ($1 || ' days')::interval`, [keepDays])
}

export async function generateInsight(triggeredBy = 'scheduler') {
  const config = await getAiInsightsConfig()
  if (!config.enabled) return { success: false, error: 'AI insights disabled' }

  let apiKey
  try {
    apiKey = await getAnthropicApiKey()
  } catch (e) {
    return { success: false, error: e.message }
  }

  const { withinBudget, usedToday } = await checkDailyBudget(config.dailyTokenBudget)
  if (!withinBudget) {
    return { success: false, error: `Daily token budget exceeded (${usedToday}/${config.dailyTokenBudget} tokens used today)` }
  }

  const monthProgress = await gatherMonthProgressData()
  const [priorPeriod, rotaVsActual, forecast, anomalies, revenueCorrelation, recentInsights] = await Promise.all([
    gatherPriorPeriodData(),
    gatherRotaVsActualData(),
    gatherForecastData(monthProgress),
    gatherEmployeeAnomalies(),
    gatherRevenueCorrelation(monthProgress),
    getRecentInsights(3),
  ])

  const { systemMsg, userMsg } = buildPrompt(monthProgress, priorPeriod, rotaVsActual, forecast, anomalies, revenueCorrelation, recentInsights)

  let result
  try {
    result = await callLlm(apiKey, systemMsg, userMsg, config.model)
  } catch (e) {
    return { success: false, error: `LLM call failed: ${e.message}` }
  }

  await saveInsight({
    content: result.content,
    model: result.model,
    input_tokens: result.input_tokens,
    output_tokens: result.output_tokens,
    data_snapshot: { monthProgress, priorPeriod, rotaVsActual, forecast, anomalies, revenueCorrelation },
    triggered_by: triggeredBy,
  })

  try {
    await cleanupOldInsights(90)
  } catch (e) {
    console.warn('[ai-insights] cleanup failed:', e.message)
  }

  return {
    success: true,
    content: result.content,
    input_tokens: result.input_tokens,
    output_tokens: result.output_tokens,
    model: result.model,
  }
}

export async function runAiInsightsGeneration() {
  try {
    const result = await generateInsight('scheduler')
    if (result.success) {
      console.log('[ai-insights] scheduled generation completed')
    } else {
      console.log('[ai-insights] scheduled generation skipped:', result.error)
    }
  } catch (e) {
    console.error('[ai-insights] scheduled generation failed:', e.message)
  }
}