management/updater/src/index.js

396 lines
14 KiB
JavaScript

import Fastify from 'fastify'
import crypto from 'crypto'
import { exec, spawn } from 'child_process'
import { promisify } from 'util'
import { readFileSync } from 'fs'
const execAsync = promisify(exec)
function spawnAsync(file, args, opts = {}) {
return new Promise((resolve, reject) => {
const child = spawn(file, args, { ...opts, shell: false })
let stdout = '', stderr = ''
child.stdout?.on('data', d => { stdout += d })
child.stderr?.on('data', d => { stderr += d })
child.on('error', reject)
child.on('close', code => {
if (code === 0) resolve({ stdout, stderr })
else reject(new Error(`Exit ${code}: ${stderr.slice(-500)}`))
})
if (opts.timeout) {
setTimeout(() => { child.kill(); reject(new Error('Timeout')) }, opts.timeout)
}
})
}
const app = Fastify({ logger: true })
// Capture the exact request bytes for webhook HMAC verification —
// re-serialising the parsed body does not round-trip Forgejo's payload.
app.addContentTypeParser('application/json', { parseAs: 'string' }, (req, body, done) => {
req.rawBody = body
try { done(null, JSON.parse(body)) } catch (err) { done(err) }
})
const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET || ''
const SSH_KEY = process.env.SSH_KEY_PATH || '/root/.ssh/id_ed25519'
const FORGEJO_URL = process.env.FORGEJO_URL || 'https://git.pterois.co.uk'
const FORGEJO_ORG = process.env.FORGEJO_ORG || 'hotel-manage-stack'
const FORGEJO_TOKEN = process.env.FORGEJO_TOKEN || ''
const AUTH_URL = process.env.AUTH_URL || 'http://10.10.10.101:3001'
const CENTRAL_AUTH_SECRET = process.env.CENTRAL_AUTH_SECRET || ''
const BACKUP_CONTAINER = process.env.BACKUP_CONTAINER || 'management-backup-1'
const RUNS_FILE = '/backups/runs.ndjson'
// Platform services: fixed IPs, never in the user-facing apps table.
// 'management' excluded from deploy — can't redeploy the container managing deploys.
const INFRA_HEALTH = {
portal: { host: '10.10.10.102', port: 3000 },
auth: { host: '10.10.10.101', port: 3001 },
settings: { host: '10.10.10.116', port: 3080 },
}
const INFRA_DEPLOY = {
portal: { host: '10.10.10.102', path: '/opt/portal' },
auth: { host: '10.10.10.101', path: '/opt/auth' },
settings: { host: '10.10.10.116', path: '/opt/settings' },
}
// App registry: fetched dynamically from auth — slug → { host, port, path }
let appRegistry = {}
async function refreshRegistry() {
try {
const res = await fetch(`${AUTH_URL}/api/auth/internal/registry`, {
headers: { Authorization: `Bearer ${CENTRAL_AUTH_SECRET}` },
signal: AbortSignal.timeout(5000),
})
if (!res.ok) { app.log.warn(`Registry fetch ${res.status}`); return }
const apps = await res.json()
const next = {}
for (const a of apps) {
if (a.internal_host && a.internal_port) {
next[a.slug] = { host: a.internal_host, port: a.internal_port, path: `/opt/${a.slug}` }
}
}
appRegistry = next
app.log.info(`Registry refreshed: ${Object.keys(next).join(', ') || '(empty)'}`)
} catch (e) {
app.log.warn(`Registry refresh failed: ${e.message}`)
}
}
function getHealthMap() {
const map = { ...INFRA_HEALTH }
for (const [slug, { host, port }] of Object.entries(appRegistry)) map[slug] = { host, port }
return map
}
function getDeployTarget(repo) {
if (INFRA_DEPLOY[repo]) return INFRA_DEPLOY[repo]
const a = appRegistry[repo]
return a ? { host: a.host, path: a.path } : null
}
function allDeployRepos() {
return [...Object.keys(INFRA_DEPLOY), ...Object.keys(appRegistry)]
}
// ── Helpers ───────────────────────────────────────────────────────────────────
const deployLog = []
const statusCache = {}
const CACHE_TTL = 5 * 60 * 1000
function verifySignature(body, signature) {
if (!WEBHOOK_SECRET) return true
// Forgejo/Gitea send a bare hex digest; GitHub prefixes it with "sha256="
const provided = (signature || '').replace(/^sha256=/, '')
const expected = crypto.createHmac('sha256', WEBHOOK_SECRET).update(body).digest('hex')
if (provided.length !== expected.length) return false
return crypto.timingSafeEqual(Buffer.from(expected), Buffer.from(provided))
}
async function sshGet(host, cmdArgs) {
try {
const { stdout } = await spawnAsync('ssh', [
'-i', SSH_KEY,
'-o', 'StrictHostKeyChecking=no',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'ConnectTimeout=2',
`root@${host}`,
...cmdArgs,
], { timeout: 8_000 })
return stdout.trim()
} catch {
return null
}
}
async function sshExec(host, command) {
return new Promise(resolve => {
const child = spawn('ssh', [
'-i', SSH_KEY,
'-o', 'StrictHostKeyChecking=no',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'ConnectTimeout=5',
`root@${host}`,
'sh', '-c', command,
], { shell: false })
let stdout = '', stderr = ''
child.stdout?.on('data', d => { stdout += d })
child.stderr?.on('data', d => { stderr += d })
const timer = setTimeout(() => {
child.kill()
resolve({ stdout, stderr: stderr + '\nTimed out after 30s', exitCode: -1 })
}, 30_000)
child.on('error', err => { clearTimeout(timer); resolve({ stdout: '', stderr: err.message, exitCode: -1 }) })
child.on('close', code => { clearTimeout(timer); resolve({ stdout, stderr, exitCode: code ?? -1 }) })
})
}
async function getForgejoCommit(repo) {
try {
const headers = { Accept: 'application/json' }
if (FORGEJO_TOKEN) headers['Authorization'] = `token ${FORGEJO_TOKEN}`
const res = await fetch(`${FORGEJO_URL}/api/v1/repos/${FORGEJO_ORG}/${repo}/commits?limit=1`, { headers })
if (!res.ok) return null
const data = await res.json()
const c = data?.[0]
if (!c) return null
return { sha: c.sha, timestamp: c.created || c.commit?.committer?.date || null }
} catch {
return null
}
}
async function refreshStatus(repo) {
const target = getDeployTarget(repo)
if (!target) return null
if (!/^\/opt\/[a-z0-9-]+$/.test(target.path)) {
app.log.error(`Rejecting suspicious path: ${target.path}`)
return null
}
const [[currentSha, currentTimestamp], latest] = await Promise.all([
Promise.all([
sshGet(target.host, ['git', '-C', target.path, 'log', '-1', '--format=%H', 'HEAD']),
sshGet(target.host, ['git', '-C', target.path, 'log', '-1', '--format=%cI', 'HEAD']),
]),
getForgejoCommit(repo),
])
const deployed = !!currentSha
const entry = {
repo,
host: target.host,
currentCommit: deployed ? currentSha.slice(0, 12) : 'not-deployed',
currentCommitAt: currentTimestamp || null,
latestCommit: latest ? latest.sha.slice(0, 12) : 'unknown',
latestCommitAt: latest?.timestamp || null,
updateAvailable: !!(deployed && latest && currentSha && latest.sha !== currentSha),
deployed,
checkedAt: new Date().toISOString(),
}
statusCache[repo] = entry
return entry
}
async function runRemote(target, repoName, cmd, label) {
if (!/^\/opt\/[a-z0-9-]+$/.test(target.path)) {
app.log.error(`Rejecting suspicious path for ${label}: ${target.path}`)
return
}
const entry = { repo: repoName, host: target.host, started: new Date().toISOString(), status: 'running', type: label }
deployLog.unshift(entry)
if (deployLog.length > 50) deployLog.pop()
try {
const { stdout } = await spawnAsync('ssh', [
'-i', SSH_KEY,
'-o', 'StrictHostKeyChecking=no',
'-o', 'UserKnownHostsFile=/dev/null',
`root@${target.host}`,
'sh', '-c', cmd,
], { timeout: 300_000 })
entry.status = 'success'
entry.output = stdout.slice(-500)
app.log.info(`${label} ${repoName} → success`)
delete statusCache[repoName]
} catch (err) {
entry.status = 'failed'
entry.error = err.message.slice(-500)
app.log.error(`${label} ${repoName} → failed`)
}
entry.finished = new Date().toISOString()
}
async function deploy(target, repoName) {
return runRemote(target, repoName, `cd ${target.path} && git pull && docker compose -f docker-compose.yml up -d --build`, 'deploy')
}
async function rebuild(target, repoName) {
return runRemote(target, repoName, `cd ${target.path} && docker compose -f docker-compose.yml up -d --build`, 'rebuild')
}
// ── Routes ────────────────────────────────────────────────────────────────────
app.get('/health', async () => ({ status: 'healthy' }))
app.get('/health-status', async () => {
const healthMap = getHealthMap()
return Promise.all(
Object.entries(healthMap).map(async ([name, { host, port }]) => {
const t0 = Date.now()
const [httpResult, freeOut, dfOut] = await Promise.all([
fetch(`http://${host}:${port}/health`, { signal: AbortSignal.timeout(3000) })
.then(() => ({ up: true, ms: Date.now() - t0 }))
.catch(() => ({ up: false, ms: null })),
sshGet(host, ['free', '-b']),
sshGet(host, ['df', '-k', '/']),
])
let ram = null
if (freeOut) {
const m = freeOut.match(/^Mem:\s+(\d+)\s+(\d+)/m)
if (m) ram = { total: +m[1], used: +m[2] }
}
let disk = null
if (dfOut) {
const row = dfOut.trim().split('\n').pop().split(/\s+/)
if (row.length >= 3) disk = { total: +row[1] * 1024, used: +row[2] * 1024 }
}
return { name, host, ...httpResult, ram, disk }
})
)
})
app.get('/deploys', async () => deployLog)
app.get('/status', async (request) => {
const single = request.query?.repo
const force = request.query?.force === 'true'
const repos = single ? [single] : allDeployRepos()
const now = Date.now()
const results = await Promise.all(repos.map(repo => {
const cached = statusCache[repo]
if (!force && cached && (now - new Date(cached.checkedAt).getTime()) < CACHE_TTL) return cached
return refreshStatus(repo)
}))
return results.filter(Boolean)
})
app.post('/deploy/:repo', async (request, reply) => {
const { repo } = request.params
const target = getDeployTarget(repo)
if (!target) return reply.status(404).send({ error: 'Unknown repo' })
const running = deployLog.find(d => d.repo === repo && d.status === 'running')
if (running) return reply.status(409).send({ error: 'Deploy already in progress' })
deploy(target, repo).catch(() => {})
return reply.status(202).send({ ok: true, deploying: repo })
})
app.post('/rebuild/:repo', async (request, reply) => {
const { repo } = request.params
const target = getDeployTarget(repo)
if (!target) return reply.status(404).send({ error: 'Unknown repo' })
const running = deployLog.find(d => d.repo === repo && d.status === 'running')
if (running) return reply.status(409).send({ error: 'Deploy already in progress' })
rebuild(target, repo).catch(() => {})
return reply.status(202).send({ ok: true, rebuilding: repo })
})
app.post('/exec', async (request, reply) => {
const { host, command } = request.body || {}
if (!host || !command || typeof command !== 'string' || command.length > 2000)
return reply.status(400).send({ error: 'host and command required' })
const knownHosts = new Set([
...Object.values(INFRA_HEALTH).map(v => v.host),
...Object.values(INFRA_DEPLOY).map(v => v.host),
...Object.values(appRegistry).map(v => v.host),
])
if (!knownHosts.has(host)) return reply.status(403).send({ error: 'unknown host' })
const result = await sshExec(host, command)
return result
})
// ── Backup routes ─────────────────────────────────────────────────────────────
function readBackupRuns() {
try {
const content = readFileSync(RUNS_FILE, 'utf8').trim()
if (!content) return []
return content.split('\n')
.filter(Boolean)
.map(line => { try { return JSON.parse(line) } catch { return null } })
.filter(Boolean)
.reverse()
} catch {
return []
}
}
app.get('/backup/runs', async () => {
return readBackupRuns().slice(0, 30)
})
app.get('/backup/status', async () => {
const runs = readBackupRuns()
if (!runs.length) return { status: 'never', last_run: null }
const last = runs[0]
return { status: last.status, last_run: last.finished, id: last.id }
})
let backupRunning = false
app.post('/backup/trigger', async (req, reply) => {
if (backupRunning) return reply.status(409).send({ error: 'Backup already running' })
backupRunning = true
reply.status(202).send({ ok: true, message: 'Backup triggered' })
spawnAsync('docker', ['exec', BACKUP_CONTAINER, 'sh', '/etc/periodic/daily/backup'], {
timeout: 600_000,
})
.then(() => { app.log.info('Manual backup completed') })
.catch(e => { app.log.error(`Manual backup failed: ${e.message}`) })
.finally(() => { backupRunning = false })
})
app.post('/webhook', { config: { rawBody: true } }, async (request, reply) => {
const sig = request.headers['x-hub-signature-256']
|| request.headers['x-forgejo-signature']
|| request.headers['x-gitea-signature']
const rawBody = request.rawBody || JSON.stringify(request.body)
if (!verifySignature(rawBody, sig)) return reply.status(401).send({ error: 'Invalid signature' })
const event = request.headers['x-gitea-event'] || request.headers['x-github-event']
if (event !== 'push') return reply.status(200).send({ ok: true, skipped: 'not a push event' })
const repoName = request.body?.repository?.name
const ref = request.body?.ref || ''
if (!ref.endsWith('/main') && !ref.endsWith('/master')) {
return reply.status(200).send({ ok: true, skipped: 'not main branch' })
}
// Refresh registry on push so a newly registered app deploys immediately
await refreshRegistry()
const target = getDeployTarget(repoName)
if (!target) return reply.status(200).send({ ok: true, skipped: 'no deploy target' })
deploy(target, repoName).catch(() => {})
return reply.status(202).send({ ok: true, deploying: repoName })
})
// ── Start ─────────────────────────────────────────────────────────────────────
await refreshRegistry()
setInterval(refreshRegistry, 5 * 60 * 1000) // keep registry fresh
await app.listen({ port: 9000, host: '0.0.0.0' })