Stack-level backup: Nextcloud WebDAV + Docker volume backup via SSH

- backup container: builds from Dockerfile (adds openssh-client, curl, jq)
  fetches Nextcloud creds from Settings API at runtime, pg_dumps all DBs
  and tars Docker volumes on each app LXC via SSH, uploads to Nextcloud
  WebDAV, writes runs.ndjson log, heartbeats Uptime Kuma
- updater: gains docker-cli, Docker socket mount, /backup/runs,
  /backup/status, /backup/trigger endpoints; reads runs.ndjson for status
- kuma_data mounted read-only into backup container for local backup
- .env.example updated with SETTINGS_URL, SETTINGS_SECRET, PG_SUPERPASS

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.example

@@ -2,6 +2,8 @@ WEBHOOK_SECRET=CHANGE_ME_match_forgejo_webhook_secret
 CENTRAL_AUTH_SECRET=CHANGE_ME_same_as_auth_service
 AUTH_URL=http://10.10.10.101:3001
 PG_SUPERPASS=CHANGE_ME_postgres_superuser_password
-BACKUP_DATABASES=auth_db noticeboard_db kitchen_db cashup_db hk_db
+BACKUP_DATABASES=auth_db noticeboard_db cashup_db hk_db maintenance_db settings_db
-BACKUP_REMOTE=user@yourserver.com:/backups/hnf-proxmox
+SETTINGS_URL=http://10.10.10.116:3080
+SETTINGS_SECRET=CHANGE_ME_same_as_settings_service
 KUMA_PUSH_URL=http://localhost:3001/api/push/KUMA_TOKEN
+# BACKUP_CONTAINER=management-backup-1

backup/Dockerfile

@@ -0,0 +1,2 @@
+FROM postgres:16-alpine
+RUN apk add --no-cache openssh-client curl jq

backup/backup.sh

@@ -1,46 +1,210 @@
 #!/bin/sh
 set -e
 
-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+TIMESTAMP=$(date -u '+%Y%m%d_%H%M%S')
-BACKUP_DIR=/backups
+RUNS_FILE=/backups/runs.ndjson
-PG_HOST=10.10.10.100
+SETTINGS_URL="${SETTINGS_URL:-http://10.10.10.116:3080}"
+SETTINGS_SECRET="${SETTINGS_SECRET:-}"
+PG_HOST="${PG_HOST:-10.10.10.100}"
+PG_SUPERPASS="${PG_SUPERPASS:-}"
+SSH_KEY="${SSH_KEY:-/root/.ssh/hotel-manage_deploy}"
 
-mkdir -p "$BACKUP_DIR/postgres/daily" "$BACKUP_DIR/postgres/weekly" "$BACKUP_DIR/volumes"
+log()  { echo "[$(date -u '+%H:%M:%S')] $1"; }
+err()  { log "ERROR: $1"; }
 
-log() { echo "[$(date '+%H:%M:%S')] $1"; }
+RUN_STARTED=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+ITEMS_FILE=$(mktemp /tmp/backup_items.XXXXXX)
+echo '[]' > "$ITEMS_FILE"
+ERRORS=0
 
-# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
+add_item() {
-log "Starting PostgreSQL backups..."
+  # args: type name status size error_msg
-for DB in $BACKUP_DATABASES; do
+  local type="$1" name="$2" status="$3" size="${4:-0}" errmsg="${5:-}"
-  OUTFILE="$BACKUP_DIR/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
+  local error_json='null'
-  PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" | gzip > "$OUTFILE"
+  [ -n "$errmsg" ] && error_json="\"$(echo "$errmsg" | sed 's/"/\\"/g')\""
-  log "  $DB → $(du -sh "$OUTFILE" | cut -f1)"
+  jq --arg t "$type" --arg n "$name" --arg s "$status" \
+     --argjson sz "$size" --argjson e "$error_json" \
+     '. += [{"type":$t,"name":$n,"status":$s,"size":$sz,"error":$e}]' \
+     "$ITEMS_FILE" > "${ITEMS_FILE}.tmp" && mv "${ITEMS_FILE}.tmp" "$ITEMS_FILE"
+}
+
+finish() {
+  local status="$1" error_msg="${2:-}"
+  local run_finished=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+  local items=$(cat "$ITEMS_FILE")
+  local error_json='null'
+  [ -n "$error_msg" ] && error_json="\"$(echo "$error_msg" | sed 's/"/\\"/g')\""
+  rm -f "$ITEMS_FILE" "${ITEMS_FILE}.tmp"
+
+  printf '{"id":"%s","started":"%s","finished":"%s","status":"%s","items":%s,"error":%s}\n' \
+    "$TIMESTAMP" "$RUN_STARTED" "$run_finished" "$status" "$items" "$error_json" \
+    >> "$RUNS_FILE"
+
+  # Keep last 30 runs
+  local lines=$(wc -l < "$RUNS_FILE" 2>/dev/null || echo 0)
+  if [ "$lines" -gt 30 ]; then
+    tail -30 "$RUNS_FILE" > "${RUNS_FILE}.tmp" && mv "${RUNS_FILE}.tmp" "$RUNS_FILE"
+  fi
+
+  # Uptime Kuma heartbeat
+  if [ -n "$KUMA_PUSH_URL" ]; then
+    local kuma_msg="Backup OK"
+    [ "$status" = "partial" ] && kuma_msg="Backup partial (${ERRORS} errors)"
+    [ "$status" = "error" ]   && kuma_msg="Backup failed: ${error_msg}"
+    wget -qO- "${KUMA_PUSH_URL}?status=up&msg=$(echo "$kuma_msg" | sed 's/ /+/g')&ping=" \
+      > /dev/null 2>&1 || true
+  fi
+
+  log "Done. Status: ${status}"
+}
+
+# ── Fetch Nextcloud credentials ───────────────────────────────────────────────
+log "Fetching Nextcloud config from Settings..."
+NC_JSON=$(curl -sf --max-time 10 \
+  -H "Authorization: Bearer ${SETTINGS_SECRET}" \
+  "${SETTINGS_URL}/settings/api/internal/integration/nextcloud" 2>/dev/null) || NC_JSON=""
+
+if [ -z "$NC_JSON" ]; then
+  err "Could not reach Settings at ${SETTINGS_URL}"
+  finish "error" "Settings service unreachable"
+  exit 1
+fi
+
+NC_BASE_URL=$(echo "$NC_JSON" | jq -r '.base_url // empty' 2>/dev/null)
+NC_USER=$(echo     "$NC_JSON" | jq -r '.username // empty'  2>/dev/null)
+NC_PASS=$(echo     "$NC_JSON" | jq -r '.password // empty'  2>/dev/null)
+NC_PATH=$(echo     "$NC_JSON" | jq -r '.backup_path // "HNF-Backups"' 2>/dev/null)
+
+if [ -z "$NC_BASE_URL" ] || [ -z "$NC_USER" ] || [ -z "$NC_PASS" ]; then
+  err "Nextcloud not configured in Settings (missing base_url, username, or password)"
+  finish "error" "Nextcloud not configured"
+  exit 1
+fi
+
+NC_DAV="${NC_BASE_URL%/}/remote.php/dav/files/${NC_USER}/${NC_PATH}"
+log "Target: ${NC_BASE_URL} → /${NC_PATH}"
+
+# Ensure Nextcloud directory tree exists (MKCOL silently ignores 405 = already exists)
+for dir in "" "/postgres" "/postgres/daily" "/postgres/weekly" "/volumes"; do
+  curl -sf -X MKCOL -u "${NC_USER}:${NC_PASS}" \
+    "${NC_DAV}${dir}" -o /dev/null 2>/dev/null || true
 done
 
-# Weekly full dump on Sundays
+# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
-if [ "$(date +%u)" = "7" ]; then
+log "PostgreSQL backups..."
-  log "Weekly full dump..."
+for DB in $BACKUP_DATABASES; do
-  PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres \
+  log "  ${DB}..."
-    | gzip > "$BACKUP_DIR/postgres/weekly/full_${TIMESTAMP}.sql.gz"
+  TARGET="${NC_DAV}/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
+  # Write to temp file so we can check pg_dump exit separately from curl
+  TMP_DUMP=$(mktemp /tmp/dump_${DB}.XXXXXX)
+  if PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" 2>/dev/null \
+       | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
+    HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
+    SIZE=$(wc -c < "$TMP_DUMP" 2>/dev/null || echo 0)
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    OK (${SIZE} bytes compressed)"
+      add_item "db" "$DB" "ok" "$SIZE"
+    else
+      err "  ${DB} upload failed (HTTP ${HTTP})"
+      add_item "db" "$DB" "error" "0" "Upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    err "  ${DB} pg_dump failed"
+    add_item "db" "$DB" "error" "0" "pg_dump failed"
+    ERRORS=$((ERRORS + 1))
+  fi
+  rm -f "$TMP_DUMP"
+done
+
+# Weekly full pg_dumpall on Sundays
+if [ "$(date -u +%u)" = "7" ]; then
+  log "  Weekly pg_dumpall..."
+  TARGET="${NC_DAV}/postgres/weekly/full_${TIMESTAMP}.sql.gz"
+  TMP_DUMP=$(mktemp /tmp/dump_full.XXXXXX)
+  if PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres 2>/dev/null \
+       | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
+    HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    Weekly dump OK"
+    else
+      err "  Weekly dump upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    err "  pg_dumpall failed"
+    ERRORS=$((ERRORS + 1))
+  fi
+  rm -f "$TMP_DUMP"
 fi
 
-# ── Retention cleanup ─────────────────────────────────────────────────────────
+# ── Volume backups via SSH ────────────────────────────────────────────────────
-find "$BACKUP_DIR/postgres/daily"  -name "*.gz" -mtime +7  -delete
+# Each line: host:docker_volume_name:backup_label
-find "$BACKUP_DIR/postgres/weekly" -name "*.gz" -mtime +28 -delete
+# Hosts that aren't deployed yet will fail silently (marked as skipped).
-log "Retention cleanup done"
+log "Volume backups..."
 
-# ── Rsync to remote ───────────────────────────────────────────────────────────
+backup_volume() {
-if [ -n "$BACKUP_REMOTE" ]; then
+  local host="$1" volname="$2" label="$3"
-  log "Syncing to $BACKUP_REMOTE..."
+  log "  ${label} @ ${host}..."
-  rsync -az --delete \
+  local target="${NC_DAV}/volumes/${label}_${TIMESTAMP}.tar.gz"
-    -e "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no" \
+  local tmp=$(mktemp /tmp/vol_${label}.XXXXXX)
-    "$BACKUP_DIR/" "$BACKUP_REMOTE"
+
-  log "Sync complete"
+  if ssh -i "$SSH_KEY" \
+        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o ConnectTimeout=5 -o BatchMode=yes \
+        "root@${host}" \
+        "docker run --rm --log-driver none -v ${volname}:/data alpine tar czf - -C /data . 2>/dev/null" \
+        > "$tmp" 2>/dev/null && [ -s "$tmp" ]; then
+    HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
+    SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    ${label} OK (${SIZE} bytes)"
+      add_item "vol" "$label" "ok" "$SIZE"
+    else
+      err "  ${label} upload failed (HTTP ${HTTP})"
+      add_item "vol" "$label" "error" "0" "Upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    log "    ${label} skipped (host offline or volume empty)"
+    add_item "vol" "$label" "skipped" "0"
+  fi
+  rm -f "$tmp"
+}
+
+backup_volume "10.10.10.117" "cashup_uploads_data"           "cashup-uploads"
+backup_volume "10.10.10.121" "maintenance_uploads_data"      "maintenance-uploads"
+backup_volume "10.10.10.113" "forecasting_recon_uploads"     "forecasting-recon"
+backup_volume "10.10.10.110" "kitchen_invoice_data"          "kitchen-invoices"
+
+# Uptime Kuma data is mounted into this container at /kuma_data
+if [ -d /kuma_data ] && [ "$(ls -A /kuma_data 2>/dev/null)" ]; then
+  log "  kuma-data (local mount)..."
+  target="${NC_DAV}/volumes/kuma-data_${TIMESTAMP}.tar.gz"
+  tmp=$(mktemp /tmp/vol_kuma.XXXXXX)
+  if tar czf "$tmp" -C /kuma_data . 2>/dev/null && [ -s "$tmp" ]; then
+    HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
+           -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
+    SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
+    if echo "$HTTP" | grep -qE '^2'; then
+      log "    kuma-data OK (${SIZE} bytes)"
+      add_item "vol" "kuma-data" "ok" "$SIZE"
+    else
+      err "  kuma-data upload failed (HTTP ${HTTP})"
+      add_item "vol" "kuma-data" "error" "0" "Upload failed (HTTP ${HTTP})"
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    log "    kuma-data skipped (empty)"
+    add_item "vol" "kuma-data" "skipped" "0"
+  fi
+  rm -f "$tmp"
 fi
 
-# ── Uptime Kuma heartbeat ─────────────────────────────────────────────────────
+# ── Finalise ──────────────────────────────────────────────────────────────────
-if [ -n "$KUMA_PUSH_URL" ]; then
+STATUS="success"
-  wget -qO- "$KUMA_PUSH_URL?status=up&msg=Backup+OK&ping=" > /dev/null 2>&1 || true
+[ "$ERRORS" -gt 0 ] && STATUS="partial"
-fi
+finish "$STATUS"
-
-log "Backup complete"

docker-compose.yml

@@ -3,7 +3,6 @@ services:
     image: louislam/uptime-kuma:2
     security_opt:
       - apparmor=unconfined
-
     volumes:
       - kuma_data:/app/data
     ports:
@@ -22,26 +21,32 @@ services:
       - FORGEJO_TOKEN=${FORGEJO_TOKEN:-}
       - AUTH_URL=${AUTH_URL:-http://10.10.10.101:3001}
       - CENTRAL_AUTH_SECRET=${CENTRAL_AUTH_SECRET}
+      - BACKUP_CONTAINER=${BACKUP_CONTAINER:-management-backup-1}
     volumes:
       - /root/.ssh:/root/.ssh:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - backup_data:/backups:ro
     ports:
       - "9000:9000"
     restart: unless-stopped
 
   backup:
-    image: postgres:16-alpine
+    build: ./backup
     security_opt:
       - apparmor=unconfined
     entrypoint: ["/bin/sh", "-c", "crond -f -l 8"]
     environment:
       - BACKUP_DATABASES=${BACKUP_DATABASES}
       - PG_SUPERPASS=${PG_SUPERPASS}
-      - BACKUP_REMOTE=${BACKUP_REMOTE}
       - KUMA_PUSH_URL=${KUMA_PUSH_URL}
+      - SETTINGS_URL=${SETTINGS_URL:-http://10.10.10.116:3080}
+      - SETTINGS_SECRET=${SETTINGS_SECRET}
+      - SSH_KEY=${SSH_KEY:-/root/.ssh/hotel-manage_deploy}
     volumes:
       - backup_data:/backups
       - ./backup/backup.sh:/etc/periodic/daily/backup:ro
       - /root/.ssh:/root/.ssh:ro
+      - kuma_data:/kuma_data:ro
     restart: unless-stopped
 
 volumes:

updater/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:20-alpine
 
-RUN apk add --no-cache openssh-client
+RUN apk add --no-cache openssh-client docker-cli
 
 WORKDIR /app
 

updater/src/index.js

@@ -2,6 +2,7 @@ import Fastify from 'fastify'
 import crypto from 'crypto'
 import { exec, spawn } from 'child_process'
 import { promisify } from 'util'
+import { readFileSync } from 'fs'
 
 const execAsync = promisify(exec)
 
@@ -37,6 +38,8 @@ const FORGEJO_ORG         = process.env.FORGEJO_ORG || 'hotel-manage-stack'
 const FORGEJO_TOKEN       = process.env.FORGEJO_TOKEN || ''
 const AUTH_URL            = process.env.AUTH_URL || 'http://10.10.10.101:3001'
 const CENTRAL_AUTH_SECRET = process.env.CENTRAL_AUTH_SECRET || ''
+const BACKUP_CONTAINER    = process.env.BACKUP_CONTAINER || 'management-backup-1'
+const RUNS_FILE           = '/backups/runs.ndjson'
 
 // Platform services: fixed IPs, never in the user-facing apps table.
 // 'management' excluded from deploy — can't redeploy the container managing deploys.
@@ -297,6 +300,49 @@ app.post('/exec', async (request, reply) => {
   return result
 })
 
+// ── Backup routes ─────────────────────────────────────────────────────────────
+
+function readBackupRuns() {
+  try {
+    const content = readFileSync(RUNS_FILE, 'utf8').trim()
+    if (!content) return []
+    return content.split('\n')
+      .filter(Boolean)
+      .map(line => { try { return JSON.parse(line) } catch { return null } })
+      .filter(Boolean)
+      .reverse()
+  } catch {
+    return []
+  }
+}
+
+app.get('/backup/runs', async () => {
+  return readBackupRuns().slice(0, 30)
+})
+
+app.get('/backup/status', async () => {
+  const runs = readBackupRuns()
+  if (!runs.length) return { status: 'never', last_run: null }
+  const last = runs[0]
+  return { status: last.status, last_run: last.finished, id: last.id }
+})
+
+let backupRunning = false
+
+app.post('/backup/trigger', async (req, reply) => {
+  if (backupRunning) return reply.status(409).send({ error: 'Backup already running' })
+
+  backupRunning = true
+  reply.status(202).send({ ok: true, message: 'Backup triggered' })
+
+  spawnAsync('docker', ['exec', BACKUP_CONTAINER, 'sh', '/etc/periodic/daily/backup'], {
+    timeout: 600_000,
+  })
+    .then(() => { app.log.info('Manual backup completed') })
+    .catch(e => { app.log.error(`Manual backup failed: ${e.message}`) })
+    .finally(() => { backupRunning = false })
+})
+
 app.post('/webhook', { config: { rawBody: true } }, async (request, reply) => {
   const sig = request.headers['x-hub-signature-256']
     || request.headers['x-forgejo-signature']