diff --git a/.env.example b/.env.example index 5379819..c91d9c1 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ WEBHOOK_SECRET=CHANGE_ME_match_forgejo_webhook_secret CENTRAL_AUTH_SECRET=CHANGE_ME_same_as_auth_service AUTH_URL=http://10.10.10.101:3001 PG_SUPERPASS=CHANGE_ME_postgres_superuser_password -BACKUP_DATABASES=auth_db noticeboard_db kitchen_db cashup_db hk_db -BACKUP_REMOTE=user@yourserver.com:/backups/hnf-proxmox +BACKUP_DATABASES=auth_db noticeboard_db cashup_db hk_db maintenance_db settings_db +SETTINGS_URL=http://10.10.10.116:3080 +SETTINGS_SECRET=CHANGE_ME_same_as_settings_service KUMA_PUSH_URL=http://localhost:3001/api/push/KUMA_TOKEN +# BACKUP_CONTAINER=management-backup-1 diff --git a/backup/Dockerfile b/backup/Dockerfile new file mode 100644 index 0000000..20d6e25 --- /dev/null +++ b/backup/Dockerfile @@ -0,0 +1,2 @@ +FROM postgres:16-alpine +RUN apk add --no-cache openssh-client curl jq diff --git a/backup/backup.sh b/backup/backup.sh index e1c0b56..0752331 100644 --- a/backup/backup.sh +++ b/backup/backup.sh @@ -1,46 +1,210 @@ #!/bin/sh set -e -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -BACKUP_DIR=/backups -PG_HOST=10.10.10.100 +TIMESTAMP=$(date -u '+%Y%m%d_%H%M%S') +RUNS_FILE=/backups/runs.ndjson +SETTINGS_URL="${SETTINGS_URL:-http://10.10.10.116:3080}" +SETTINGS_SECRET="${SETTINGS_SECRET:-}" +PG_HOST="${PG_HOST:-10.10.10.100}" +PG_SUPERPASS="${PG_SUPERPASS:-}" +SSH_KEY="${SSH_KEY:-/root/.ssh/hotel-manage_deploy}" -mkdir -p "$BACKUP_DIR/postgres/daily" "$BACKUP_DIR/postgres/weekly" "$BACKUP_DIR/volumes" +log() { echo "[$(date -u '+%H:%M:%S')] $1"; } +err() { log "ERROR: $1"; } -log() { echo "[$(date '+%H:%M:%S')] $1"; } +RUN_STARTED=$(date -u '+%Y-%m-%dT%H:%M:%SZ') +ITEMS_FILE=$(mktemp /tmp/backup_items.XXXXXX) +echo '[]' > "$ITEMS_FILE" +ERRORS=0 -# ── PostgreSQL dumps ────────────────────────────────────────────────────────── -log "Starting PostgreSQL backups..." -for DB in $BACKUP_DATABASES; do - OUTFILE="$BACKUP_DIR/postgres/daily/${DB}_${TIMESTAMP}.sql.gz" - PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" | gzip > "$OUTFILE" - log " $DB → $(du -sh "$OUTFILE" | cut -f1)" +add_item() { + # args: type name status size error_msg + local type="$1" name="$2" status="$3" size="${4:-0}" errmsg="${5:-}" + local error_json='null' + [ -n "$errmsg" ] && error_json="\"$(echo "$errmsg" | sed 's/"/\\"/g')\"" + jq --arg t "$type" --arg n "$name" --arg s "$status" \ + --argjson sz "$size" --argjson e "$error_json" \ + '. += [{"type":$t,"name":$n,"status":$s,"size":$sz,"error":$e}]' \ + "$ITEMS_FILE" > "${ITEMS_FILE}.tmp" && mv "${ITEMS_FILE}.tmp" "$ITEMS_FILE" +} + +finish() { + local status="$1" error_msg="${2:-}" + local run_finished=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + local items=$(cat "$ITEMS_FILE") + local error_json='null' + [ -n "$error_msg" ] && error_json="\"$(echo "$error_msg" | sed 's/"/\\"/g')\"" + rm -f "$ITEMS_FILE" "${ITEMS_FILE}.tmp" + + printf '{"id":"%s","started":"%s","finished":"%s","status":"%s","items":%s,"error":%s}\n' \ + "$TIMESTAMP" "$RUN_STARTED" "$run_finished" "$status" "$items" "$error_json" \ + >> "$RUNS_FILE" + + # Keep last 30 runs + local lines=$(wc -l < "$RUNS_FILE" 2>/dev/null || echo 0) + if [ "$lines" -gt 30 ]; then + tail -30 "$RUNS_FILE" > "${RUNS_FILE}.tmp" && mv "${RUNS_FILE}.tmp" "$RUNS_FILE" + fi + + # Uptime Kuma heartbeat + if [ -n "$KUMA_PUSH_URL" ]; then + local kuma_msg="Backup OK" + [ "$status" = "partial" ] && kuma_msg="Backup partial (${ERRORS} errors)" + [ "$status" = "error" ] && kuma_msg="Backup failed: ${error_msg}" + wget -qO- "${KUMA_PUSH_URL}?status=up&msg=$(echo "$kuma_msg" | sed 's/ /+/g')&ping=" \ + > /dev/null 2>&1 || true + fi + + log "Done. Status: ${status}" +} + +# ── Fetch Nextcloud credentials ─────────────────────────────────────────────── +log "Fetching Nextcloud config from Settings..." +NC_JSON=$(curl -sf --max-time 10 \ + -H "Authorization: Bearer ${SETTINGS_SECRET}" \ + "${SETTINGS_URL}/settings/api/internal/integration/nextcloud" 2>/dev/null) || NC_JSON="" + +if [ -z "$NC_JSON" ]; then + err "Could not reach Settings at ${SETTINGS_URL}" + finish "error" "Settings service unreachable" + exit 1 +fi + +NC_BASE_URL=$(echo "$NC_JSON" | jq -r '.base_url // empty' 2>/dev/null) +NC_USER=$(echo "$NC_JSON" | jq -r '.username // empty' 2>/dev/null) +NC_PASS=$(echo "$NC_JSON" | jq -r '.password // empty' 2>/dev/null) +NC_PATH=$(echo "$NC_JSON" | jq -r '.backup_path // "HNF-Backups"' 2>/dev/null) + +if [ -z "$NC_BASE_URL" ] || [ -z "$NC_USER" ] || [ -z "$NC_PASS" ]; then + err "Nextcloud not configured in Settings (missing base_url, username, or password)" + finish "error" "Nextcloud not configured" + exit 1 +fi + +NC_DAV="${NC_BASE_URL%/}/remote.php/dav/files/${NC_USER}/${NC_PATH}" +log "Target: ${NC_BASE_URL} → /${NC_PATH}" + +# Ensure Nextcloud directory tree exists (MKCOL silently ignores 405 = already exists) +for dir in "" "/postgres" "/postgres/daily" "/postgres/weekly" "/volumes"; do + curl -sf -X MKCOL -u "${NC_USER}:${NC_PASS}" \ + "${NC_DAV}${dir}" -o /dev/null 2>/dev/null || true done -# Weekly full dump on Sundays -if [ "$(date +%u)" = "7" ]; then - log "Weekly full dump..." - PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres \ - | gzip > "$BACKUP_DIR/postgres/weekly/full_${TIMESTAMP}.sql.gz" +# ── PostgreSQL dumps ────────────────────────────────────────────────────────── +log "PostgreSQL backups..." +for DB in $BACKUP_DATABASES; do + log " ${DB}..." + TARGET="${NC_DAV}/postgres/daily/${DB}_${TIMESTAMP}.sql.gz" + # Write to temp file so we can check pg_dump exit separately from curl + TMP_DUMP=$(mktemp /tmp/dump_${DB}.XXXXXX) + if PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" 2>/dev/null \ + | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then + HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \ + -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000") + SIZE=$(wc -c < "$TMP_DUMP" 2>/dev/null || echo 0) + if echo "$HTTP" | grep -qE '^2'; then + log " OK (${SIZE} bytes compressed)" + add_item "db" "$DB" "ok" "$SIZE" + else + err " ${DB} upload failed (HTTP ${HTTP})" + add_item "db" "$DB" "error" "0" "Upload failed (HTTP ${HTTP})" + ERRORS=$((ERRORS + 1)) + fi + else + err " ${DB} pg_dump failed" + add_item "db" "$DB" "error" "0" "pg_dump failed" + ERRORS=$((ERRORS + 1)) + fi + rm -f "$TMP_DUMP" +done + +# Weekly full pg_dumpall on Sundays +if [ "$(date -u +%u)" = "7" ]; then + log " Weekly pg_dumpall..." + TARGET="${NC_DAV}/postgres/weekly/full_${TIMESTAMP}.sql.gz" + TMP_DUMP=$(mktemp /tmp/dump_full.XXXXXX) + if PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres 2>/dev/null \ + | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then + HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \ + -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000") + if echo "$HTTP" | grep -qE '^2'; then + log " Weekly dump OK" + else + err " Weekly dump upload failed (HTTP ${HTTP})" + ERRORS=$((ERRORS + 1)) + fi + else + err " pg_dumpall failed" + ERRORS=$((ERRORS + 1)) + fi + rm -f "$TMP_DUMP" fi -# ── Retention cleanup ───────────────────────────────────────────────────────── -find "$BACKUP_DIR/postgres/daily" -name "*.gz" -mtime +7 -delete -find "$BACKUP_DIR/postgres/weekly" -name "*.gz" -mtime +28 -delete -log "Retention cleanup done" +# ── Volume backups via SSH ──────────────────────────────────────────────────── +# Each line: host:docker_volume_name:backup_label +# Hosts that aren't deployed yet will fail silently (marked as skipped). +log "Volume backups..." -# ── Rsync to remote ─────────────────────────────────────────────────────────── -if [ -n "$BACKUP_REMOTE" ]; then - log "Syncing to $BACKUP_REMOTE..." - rsync -az --delete \ - -e "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no" \ - "$BACKUP_DIR/" "$BACKUP_REMOTE" - log "Sync complete" +backup_volume() { + local host="$1" volname="$2" label="$3" + log " ${label} @ ${host}..." + local target="${NC_DAV}/volumes/${label}_${TIMESTAMP}.tar.gz" + local tmp=$(mktemp /tmp/vol_${label}.XXXXXX) + + if ssh -i "$SSH_KEY" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 -o BatchMode=yes \ + "root@${host}" \ + "docker run --rm --log-driver none -v ${volname}:/data alpine tar czf - -C /data . 2>/dev/null" \ + > "$tmp" 2>/dev/null && [ -s "$tmp" ]; then + HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \ + -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000") + SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0) + if echo "$HTTP" | grep -qE '^2'; then + log " ${label} OK (${SIZE} bytes)" + add_item "vol" "$label" "ok" "$SIZE" + else + err " ${label} upload failed (HTTP ${HTTP})" + add_item "vol" "$label" "error" "0" "Upload failed (HTTP ${HTTP})" + ERRORS=$((ERRORS + 1)) + fi + else + log " ${label} skipped (host offline or volume empty)" + add_item "vol" "$label" "skipped" "0" + fi + rm -f "$tmp" +} + +backup_volume "10.10.10.117" "cashup_uploads_data" "cashup-uploads" +backup_volume "10.10.10.121" "maintenance_uploads_data" "maintenance-uploads" +backup_volume "10.10.10.113" "forecasting_recon_uploads" "forecasting-recon" +backup_volume "10.10.10.110" "kitchen_invoice_data" "kitchen-invoices" + +# Uptime Kuma data is mounted into this container at /kuma_data +if [ -d /kuma_data ] && [ "$(ls -A /kuma_data 2>/dev/null)" ]; then + log " kuma-data (local mount)..." + target="${NC_DAV}/volumes/kuma-data_${TIMESTAMP}.tar.gz" + tmp=$(mktemp /tmp/vol_kuma.XXXXXX) + if tar czf "$tmp" -C /kuma_data . 2>/dev/null && [ -s "$tmp" ]; then + HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \ + -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000") + SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0) + if echo "$HTTP" | grep -qE '^2'; then + log " kuma-data OK (${SIZE} bytes)" + add_item "vol" "kuma-data" "ok" "$SIZE" + else + err " kuma-data upload failed (HTTP ${HTTP})" + add_item "vol" "kuma-data" "error" "0" "Upload failed (HTTP ${HTTP})" + ERRORS=$((ERRORS + 1)) + fi + else + log " kuma-data skipped (empty)" + add_item "vol" "kuma-data" "skipped" "0" + fi + rm -f "$tmp" fi -# ── Uptime Kuma heartbeat ───────────────────────────────────────────────────── -if [ -n "$KUMA_PUSH_URL" ]; then - wget -qO- "$KUMA_PUSH_URL?status=up&msg=Backup+OK&ping=" > /dev/null 2>&1 || true -fi - -log "Backup complete" +# ── Finalise ────────────────────────────────────────────────────────────────── +STATUS="success" +[ "$ERRORS" -gt 0 ] && STATUS="partial" +finish "$STATUS" diff --git a/docker-compose.yml b/docker-compose.yml index 7221207..f2c0e6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ services: image: louislam/uptime-kuma:2 security_opt: - apparmor=unconfined - volumes: - kuma_data:/app/data ports: @@ -22,26 +21,32 @@ services: - FORGEJO_TOKEN=${FORGEJO_TOKEN:-} - AUTH_URL=${AUTH_URL:-http://10.10.10.101:3001} - CENTRAL_AUTH_SECRET=${CENTRAL_AUTH_SECRET} + - BACKUP_CONTAINER=${BACKUP_CONTAINER:-management-backup-1} volumes: - /root/.ssh:/root/.ssh:ro + - /var/run/docker.sock:/var/run/docker.sock + - backup_data:/backups:ro ports: - "9000:9000" restart: unless-stopped backup: - image: postgres:16-alpine + build: ./backup security_opt: - apparmor=unconfined entrypoint: ["/bin/sh", "-c", "crond -f -l 8"] environment: - BACKUP_DATABASES=${BACKUP_DATABASES} - PG_SUPERPASS=${PG_SUPERPASS} - - BACKUP_REMOTE=${BACKUP_REMOTE} - KUMA_PUSH_URL=${KUMA_PUSH_URL} + - SETTINGS_URL=${SETTINGS_URL:-http://10.10.10.116:3080} + - SETTINGS_SECRET=${SETTINGS_SECRET} + - SSH_KEY=${SSH_KEY:-/root/.ssh/hotel-manage_deploy} volumes: - backup_data:/backups - ./backup/backup.sh:/etc/periodic/daily/backup:ro - /root/.ssh:/root/.ssh:ro + - kuma_data:/kuma_data:ro restart: unless-stopped volumes: diff --git a/updater/Dockerfile b/updater/Dockerfile index 0c3fa06..da388b4 100644 --- a/updater/Dockerfile +++ b/updater/Dockerfile @@ -1,6 +1,6 @@ FROM node:20-alpine -RUN apk add --no-cache openssh-client +RUN apk add --no-cache openssh-client docker-cli WORKDIR /app diff --git a/updater/src/index.js b/updater/src/index.js index d13726e..345a31d 100644 --- a/updater/src/index.js +++ b/updater/src/index.js @@ -2,6 +2,7 @@ import Fastify from 'fastify' import crypto from 'crypto' import { exec, spawn } from 'child_process' import { promisify } from 'util' +import { readFileSync } from 'fs' const execAsync = promisify(exec) @@ -37,6 +38,8 @@ const FORGEJO_ORG = process.env.FORGEJO_ORG || 'hotel-manage-stack' const FORGEJO_TOKEN = process.env.FORGEJO_TOKEN || '' const AUTH_URL = process.env.AUTH_URL || 'http://10.10.10.101:3001' const CENTRAL_AUTH_SECRET = process.env.CENTRAL_AUTH_SECRET || '' +const BACKUP_CONTAINER = process.env.BACKUP_CONTAINER || 'management-backup-1' +const RUNS_FILE = '/backups/runs.ndjson' // Platform services: fixed IPs, never in the user-facing apps table. // 'management' excluded from deploy — can't redeploy the container managing deploys. @@ -297,6 +300,49 @@ app.post('/exec', async (request, reply) => { return result }) +// ── Backup routes ───────────────────────────────────────────────────────────── + +function readBackupRuns() { + try { + const content = readFileSync(RUNS_FILE, 'utf8').trim() + if (!content) return [] + return content.split('\n') + .filter(Boolean) + .map(line => { try { return JSON.parse(line) } catch { return null } }) + .filter(Boolean) + .reverse() + } catch { + return [] + } +} + +app.get('/backup/runs', async () => { + return readBackupRuns().slice(0, 30) +}) + +app.get('/backup/status', async () => { + const runs = readBackupRuns() + if (!runs.length) return { status: 'never', last_run: null } + const last = runs[0] + return { status: last.status, last_run: last.finished, id: last.id } +}) + +let backupRunning = false + +app.post('/backup/trigger', async (req, reply) => { + if (backupRunning) return reply.status(409).send({ error: 'Backup already running' }) + + backupRunning = true + reply.status(202).send({ ok: true, message: 'Backup triggered' }) + + spawnAsync('docker', ['exec', BACKUP_CONTAINER, 'sh', '/etc/periodic/daily/backup'], { + timeout: 600_000, + }) + .then(() => { app.log.info('Manual backup completed') }) + .catch(e => { app.log.error(`Manual backup failed: ${e.message}`) }) + .finally(() => { backupRunning = false }) +}) + app.post('/webhook', { config: { rawBody: true } }, async (request, reply) => { const sig = request.headers['x-hub-signature-256'] || request.headers['x-forgejo-signature']