Stack-level backup: Nextcloud WebDAV + Docker volume backup via SSH

- backup container: builds from Dockerfile (adds openssh-client, curl, jq) fetches Nextcloud creds from Settings API at runtime, pg_dumps all DBs and tars Docker volumes on each app LXC via SSH, uploads to Nextcloud WebDAV, writes runs.ndjson log, heartbeats Uptime Kuma - updater: gains docker-cli, Docker socket mount, /backup/runs, /backup/status, /backup/trigger endpoints; reads runs.ndjson for status - kuma_data mounted read-only into backup container for local backup - .env.example updated with SETTINGS_URL, SETTINGS_SECRET, PG_SUPERPASS Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-07-14 11:03:59 +00:00 · 2026-07-14 11:03:59 +00:00 · f46fe07ca8
commit f46fe07ca8
parent cc5fc99242
6 changed files with 258 additions and 39 deletions
--- a/.env.example
+++ b/.env.example
@ -2,6 +2,8 @@ WEBHOOK_SECRET=CHANGE_ME_match_forgejo_webhook_secret
 CENTRAL_AUTH_SECRET=CHANGE_ME_same_as_auth_service
 AUTH_URL=http://10.10.10.101:3001
 PG_SUPERPASS=CHANGE_ME_postgres_superuser_password
-BACKUP_DATABASES=auth_db noticeboard_db kitchen_db cashup_db hk_db
-BACKUP_REMOTE=user@yourserver.com:/backups/hnf-proxmox
+BACKUP_DATABASES=auth_db noticeboard_db cashup_db hk_db maintenance_db settings_db
+SETTINGS_URL=http://10.10.10.116:3080
+SETTINGS_SECRET=CHANGE_ME_same_as_settings_service
 KUMA_PUSH_URL=http://localhost:3001/api/push/KUMA_TOKEN
+# BACKUP_CONTAINER=management-backup-1
--- a/backup/Dockerfile
+++ b/backup/Dockerfile
@ -0,0 +1,2 @@
+FROM postgres:16-alpine
+RUN apk add --no-cache openssh-client curl jq
--- a/backup/backup.sh
+++ b/backup/backup.sh
@ -1,46 +1,210 @@
 #!/bin/sh
 set -e

-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
-BACKUP_DIR=/backups
-PG_HOST=10.10.10.100
+TIMESTAMP=$(date -u '+%Y%m%d_%H%M%S')
+RUNS_FILE=/backups/runs.ndjson
+SETTINGS_URL="${SETTINGS_URL:-http://10.10.10.116:3080}"
+SETTINGS_SECRET="${SETTINGS_SECRET:-}"
+PG_HOST="${PG_HOST:-10.10.10.100}"
+PG_SUPERPASS="${PG_SUPERPASS:-}"
+SSH_KEY="${SSH_KEY:-/root/.ssh/hotel-manage_deploy}"

-mkdir -p "$BACKUP_DIR/postgres/daily" "$BACKUP_DIR/postgres/weekly" "$BACKUP_DIR/volumes"
+log()  { echo "[$(date -u '+%H:%M:%S')] $1"; }
+err()  { log "ERROR: $1"; }

-log() { echo "[$(date '+%H:%M:%S')] $1"; }
+RUN_STARTED=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+ITEMS_FILE=$(mktemp /tmp/backup_items.XXXXXX)
+echo '[]' > "$ITEMS_FILE"
+ERRORS=0

-# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
-log "Starting PostgreSQL backups..."
-for DB in $BACKUP_DATABASES; do
-  OUTFILE="$BACKUP_DIR/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
-  PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" | gzip > "$OUTFILE"
-  log "  $DB → $(du -sh "$OUTFILE" | cut -f1)"
+add_item() {
+  # args: type name status size error_msg
+  local type="$1" name="$2" status="$3" size="${4:-0}" errmsg="${5:-}"
+  local error_json='null'
+  [ -n "$errmsg" ] && error_json="\"$(echo "$errmsg" | sed 's/"/\\"/g')\""
+  jq --arg t "$type" --arg n "$name" --arg s "$status" \
+     --argjson sz "$size" --argjson e "$error_json" \
+     '. += [{"type":$t,"name":$n,"status":$s,"size":$sz,"error":$e}]' \
+     "$ITEMS_FILE" > "${ITEMS_FILE}.tmp" && mv "${ITEMS_FILE}.tmp" "$ITEMS_FILE"
+}
+
+finish() {
+  local status="$1" error_msg="${2:-}"
+  local run_finished=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+  local items=$(cat "$ITEMS_FILE")
+  local error_json='null'
+  [ -n "$error_msg" ] && error_json="\"$(echo "$error_msg" | sed 's/"/\\"/g')\""
+  rm -f "$ITEMS_FILE" "${ITEMS_FILE}.tmp"
+
+  printf '{"id":"%s","started":"%s","finished":"%s","status":"%s","items":%s,"error":%s}\n' \
+    "$TIMESTAMP" "$RUN_STARTED" "$run_finished" "$status" "$items" "$error_json" \
+    >> "$RUNS_FILE"
+
+  # Keep last 30 runs
+  local lines=$(wc -l < "$RUNS_FILE" 2>/dev/null || echo 0)
+  if [ "$lines" -gt 30 ]; then
+    tail -30 "$RUNS_FILE" > "${RUNS_FILE}.tmp" && mv "${RUNS_FILE}.tmp" "$RUNS_FILE"
+  fi
+
+  # Uptime Kuma heartbeat
+  if [ -n "$KUMA_PUSH_URL" ]; then
+    local kuma_msg="Backup OK"
+    [ "$status" = "partial" ] && kuma_msg="Backup partial (${ERRORS} errors)"
+    [ "$status" = "error" ]   && kuma_msg="Backup failed: ${error_msg}"
+    wget -qO- "${KUMA_PUSH_URL}?status=up&msg=$(echo "$kuma_msg" | sed 's/ /+/g')&ping=" \
+      > /dev/null 2>&1 || true
+  fi
+
+  log "Done. Status: ${status}"
+}
+
+# ── Fetch Nextcloud credentials ───────────────────────────────────────────────
+log "Fetching Nextcloud config from Settings..."
+NC_JSON=$(curl -sf --max-time 10 \
+  -H "Authorization: Bearer ${SETTINGS_SECRET}" \
+  "${SETTINGS_URL}/settings/api/internal/integration/nextcloud" 2>/dev/null) || NC_JSON=""
+
+if [ -z "$NC_JSON" ]; then
+  err "Could not reach Settings at ${SETTINGS_URL}"
+  finish "error" "Settings service unreachable"
+  exit 1
+fi
+
+NC_BASE_URL=$(echo "$NC_JSON" | jq -r '.base_url // empty' 2>/dev/null)
+NC_USER=$(echo     "$NC_JSON" | jq -r '.username // empty'  2>/dev/null)
+NC_PASS=$(echo     "$NC_JSON" | jq -r '.password // empty'  2>/dev/null)
+NC_PATH=$(echo     "$NC_JSON" | jq -r '.backup_path // "HNF-Backups"' 2>/dev/null)
+
+if [ -z "$NC_BASE_URL" ] || [ -z "$NC_USER" ] || [ -z "$NC_PASS" ]; then
+  err "Nextcloud not configured in Settings (missing base_url, username, or password)"
+  finish "error" "Nextcloud not configured"
+  exit 1
+fi
+
+NC_DAV="${NC_BASE_URL%/}/remote.php/dav/files/${NC_USER}/${NC_PATH}"
+log "Target: ${NC_BASE_URL} → /${NC_PATH}"
+
+# Ensure Nextcloud directory tree exists (MKCOL silently ignores 405 = already exists)
+for dir in "" "/postgres" "/postgres/daily" "/postgres/weekly" "/volumes"; do
+  curl -sf -X MKCOL -u "${NC_USER}:${NC_PASS}" \
+    "${NC_DAV}${dir}" -o /dev/null 2>/dev/null || true
 done

-# Weekly full dump on Sundays
-if [ "$(date +%u)" = "7" ]; then
-  log "Weekly full dump..."
-  PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres \
-    | gzip > "$BACKUP_DIR/postgres/weekly/full_${TIMESTAMP}.sql.gz"
+# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
+log "PostgreSQL backups..."
+for DB in $BACKUP_DATABASES; do
+  log "  ${DB}..."
+  TARGET="${NC_DAV}/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
+  # Write to temp file so we can check pg_dump exit separately from curl
+  TMP_DUMP=$(mktemp /tmp/dump_${DB}.XXXXXX)
+  if PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" 2>/dev/null \
+       | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
+    HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
+    SIZE=$(wc -c < "$TMP_DUMP" 2>/dev/null || echo 0)
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    OK (${SIZE} bytes compressed)"
+      add_item "db" "$DB" "ok" "$SIZE"
+    else
+      err "  ${DB} upload failed (HTTP ${HTTP})"
+      add_item "db" "$DB" "error" "0" "Upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    err "  ${DB} pg_dump failed"
+    add_item "db" "$DB" "error" "0" "pg_dump failed"
+    ERRORS=$((ERRORS + 1))
+  fi
+  rm -f "$TMP_DUMP"
+done
+
+# Weekly full pg_dumpall on Sundays
+if [ "$(date -u +%u)" = "7" ]; then
+  log "  Weekly pg_dumpall..."
+  TARGET="${NC_DAV}/postgres/weekly/full_${TIMESTAMP}.sql.gz"
+  TMP_DUMP=$(mktemp /tmp/dump_full.XXXXXX)
+  if PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres 2>/dev/null \
+       | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
+    HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    Weekly dump OK"
+    else
+      err "  Weekly dump upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    err "  pg_dumpall failed"
+    ERRORS=$((ERRORS + 1))
+  fi
+  rm -f "$TMP_DUMP"
 fi

-# ── Retention cleanup ─────────────────────────────────────────────────────────
-find "$BACKUP_DIR/postgres/daily"  -name "*.gz" -mtime +7  -delete
-find "$BACKUP_DIR/postgres/weekly" -name "*.gz" -mtime +28 -delete
-log "Retention cleanup done"
+# ── Volume backups via SSH ────────────────────────────────────────────────────
+# Each line: host:docker_volume_name:backup_label
+# Hosts that aren't deployed yet will fail silently (marked as skipped).
+log "Volume backups..."

-# ── Rsync to remote ───────────────────────────────────────────────────────────
-if [ -n "$BACKUP_REMOTE" ]; then
-  log "Syncing to $BACKUP_REMOTE..."
-  rsync -az --delete \
-    -e "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no" \
-    "$BACKUP_DIR/" "$BACKUP_REMOTE"
-  log "Sync complete"
+backup_volume() {
+  local host="$1" volname="$2" label="$3"
+  log "  ${label} @ ${host}..."
+  local target="${NC_DAV}/volumes/${label}_${TIMESTAMP}.tar.gz"
+  local tmp=$(mktemp /tmp/vol_${label}.XXXXXX)
+
+  if ssh -i "$SSH_KEY" \
+        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o ConnectTimeout=5 -o BatchMode=yes \
+        "root@${host}" \
+        "docker run --rm --log-driver none -v ${volname}:/data alpine tar czf - -C /data . 2>/dev/null" \
+        > "$tmp" 2>/dev/null && [ -s "$tmp" ]; then
+    HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
+    SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    ${label} OK (${SIZE} bytes)"
+      add_item "vol" "$label" "ok" "$SIZE"
+    else
+      err "  ${label} upload failed (HTTP ${HTTP})"
+      add_item "vol" "$label" "error" "0" "Upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    log "    ${label} skipped (host offline or volume empty)"
+    add_item "vol" "$label" "skipped" "0"
+  fi
+  rm -f "$tmp"
+}
+
+backup_volume "10.10.10.117" "cashup_uploads_data"           "cashup-uploads"
+backup_volume "10.10.10.121" "maintenance_uploads_data"      "maintenance-uploads"
+backup_volume "10.10.10.113" "forecasting_recon_uploads"     "forecasting-recon"
+backup_volume "10.10.10.110" "kitchen_invoice_data"          "kitchen-invoices"
+
+# Uptime Kuma data is mounted into this container at /kuma_data
+if [ -d /kuma_data ] && [ "$(ls -A /kuma_data 2>/dev/null)" ]; then
+  log "  kuma-data (local mount)..."
+  target="${NC_DAV}/volumes/kuma-data_${TIMESTAMP}.tar.gz"
+  tmp=$(mktemp /tmp/vol_kuma.XXXXXX)
+  if tar czf "$tmp" -C /kuma_data . 2>/dev/null && [ -s "$tmp" ]; then
+    HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
+    SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    kuma-data OK (${SIZE} bytes)"
+      add_item "vol" "kuma-data" "ok" "$SIZE"
+    else
+      err "  kuma-data upload failed (HTTP ${HTTP})"
+      add_item "vol" "kuma-data" "error" "0" "Upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    log "    kuma-data skipped (empty)"
+    add_item "vol" "kuma-data" "skipped" "0"
+  fi
+  rm -f "$tmp"
 fi

-# ── Uptime Kuma heartbeat ─────────────────────────────────────────────────────
-if [ -n "$KUMA_PUSH_URL" ]; then
-  wget -qO- "$KUMA_PUSH_URL?status=up&msg=Backup+OK&ping=" > /dev/null 2>&1 || true
-fi
-
-log "Backup complete"
+# ── Finalise ──────────────────────────────────────────────────────────────────
+STATUS="success"
+[ "$ERRORS" -gt 0 ] && STATUS="partial"
+finish "$STATUS"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -3,7 +3,6 @@ services:
    image: louislam/uptime-kuma:2
    security_opt:
      - apparmor=unconfined
-
    volumes:
      - kuma_data:/app/data
    ports:
@ -22,26 +21,32 @@ services:
      - FORGEJO_TOKEN=${FORGEJO_TOKEN:-}
      - AUTH_URL=${AUTH_URL:-http://10.10.10.101:3001}
      - CENTRAL_AUTH_SECRET=${CENTRAL_AUTH_SECRET}
+      - BACKUP_CONTAINER=${BACKUP_CONTAINER:-management-backup-1}
    volumes:
      - /root/.ssh:/root/.ssh:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - backup_data:/backups:ro
    ports:
      - "9000:9000"
    restart: unless-stopped

  backup:
-    image: postgres:16-alpine
+    build: ./backup
    security_opt:
      - apparmor=unconfined
    entrypoint: ["/bin/sh", "-c", "crond -f -l 8"]
    environment:
      - BACKUP_DATABASES=${BACKUP_DATABASES}
      - PG_SUPERPASS=${PG_SUPERPASS}
-      - BACKUP_REMOTE=${BACKUP_REMOTE}
      - KUMA_PUSH_URL=${KUMA_PUSH_URL}
+      - SETTINGS_URL=${SETTINGS_URL:-http://10.10.10.116:3080}
+      - SETTINGS_SECRET=${SETTINGS_SECRET}
+      - SSH_KEY=${SSH_KEY:-/root/.ssh/hotel-manage_deploy}
    volumes:
      - backup_data:/backups
      - ./backup/backup.sh:/etc/periodic/daily/backup:ro
      - /root/.ssh:/root/.ssh:ro
+      - kuma_data:/kuma_data:ro
    restart: unless-stopped

 volumes:
--- a/updater/Dockerfile
+++ b/updater/Dockerfile
@ -1,6 +1,6 @@
 FROM node:20-alpine

-RUN apk add --no-cache openssh-client
+RUN apk add --no-cache openssh-client docker-cli

 WORKDIR /app

--- a/updater/src/index.js
+++ b/updater/src/index.js
@ -2,6 +2,7 @@ import Fastify from 'fastify'
 import crypto from 'crypto'
 import { exec, spawn } from 'child_process'
 import { promisify } from 'util'
+import { readFileSync } from 'fs'

 const execAsync = promisify(exec)

@ -37,6 +38,8 @@ const FORGEJO_ORG         = process.env.FORGEJO_ORG || 'hotel-manage-stack'
 const FORGEJO_TOKEN       = process.env.FORGEJO_TOKEN || ''
 const AUTH_URL            = process.env.AUTH_URL || 'http://10.10.10.101:3001'
 const CENTRAL_AUTH_SECRET = process.env.CENTRAL_AUTH_SECRET || ''
+const BACKUP_CONTAINER    = process.env.BACKUP_CONTAINER || 'management-backup-1'
+const RUNS_FILE           = '/backups/runs.ndjson'

 // Platform services: fixed IPs, never in the user-facing apps table.
 // 'management' excluded from deploy — can't redeploy the container managing deploys.
@ -297,6 +300,49 @@ app.post('/exec', async (request, reply) => {
  return result
 })

+// ── Backup routes ─────────────────────────────────────────────────────────────
+
+function readBackupRuns() {
+  try {
+    const content = readFileSync(RUNS_FILE, 'utf8').trim()
+    if (!content) return []
+    return content.split('\n')
+      .filter(Boolean)
+      .map(line => { try { return JSON.parse(line) } catch { return null } })
+      .filter(Boolean)
+      .reverse()
+  } catch {
+    return []
+  }
+}
+
+app.get('/backup/runs', async () => {
+  return readBackupRuns().slice(0, 30)
+})
+
+app.get('/backup/status', async () => {
+  const runs = readBackupRuns()
+  if (!runs.length) return { status: 'never', last_run: null }
+  const last = runs[0]
+  return { status: last.status, last_run: last.finished, id: last.id }
+})
+
+let backupRunning = false
+
+app.post('/backup/trigger', async (req, reply) => {
+  if (backupRunning) return reply.status(409).send({ error: 'Backup already running' })
+
+  backupRunning = true
+  reply.status(202).send({ ok: true, message: 'Backup triggered' })
+
+  spawnAsync('docker', ['exec', BACKUP_CONTAINER, 'sh', '/etc/periodic/daily/backup'], {
+    timeout: 600_000,
+  })
+    .then(() => { app.log.info('Manual backup completed') })
+    .catch(e => { app.log.error(`Manual backup failed: ${e.message}`) })
+    .finally(() => { backupRunning = false })
+})
+
 app.post('/webhook', { config: { rawBody: true } }, async (request, reply) => {
  const sig = request.headers['x-hub-signature-256']
    || request.headers['x-forgejo-signature']