Stack-level backup: Nextcloud WebDAV + Docker volume backup via SSH
- backup container: builds from Dockerfile (adds openssh-client, curl, jq) fetches Nextcloud creds from Settings API at runtime, pg_dumps all DBs and tars Docker volumes on each app LXC via SSH, uploads to Nextcloud WebDAV, writes runs.ndjson log, heartbeats Uptime Kuma - updater: gains docker-cli, Docker socket mount, /backup/runs, /backup/status, /backup/trigger endpoints; reads runs.ndjson for status - kuma_data mounted read-only into backup container for local backup - .env.example updated with SETTINGS_URL, SETTINGS_SECRET, PG_SUPERPASS Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cc5fc99242
commit
f46fe07ca8
6 changed files with 258 additions and 39 deletions
2
backup/Dockerfile
Normal file
2
backup/Dockerfile
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
FROM postgres:16-alpine
|
||||
RUN apk add --no-cache openssh-client curl jq
|
||||
230
backup/backup.sh
230
backup/backup.sh
|
|
@ -1,46 +1,210 @@
|
|||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
BACKUP_DIR=/backups
|
||||
PG_HOST=10.10.10.100
|
||||
TIMESTAMP=$(date -u '+%Y%m%d_%H%M%S')
|
||||
RUNS_FILE=/backups/runs.ndjson
|
||||
SETTINGS_URL="${SETTINGS_URL:-http://10.10.10.116:3080}"
|
||||
SETTINGS_SECRET="${SETTINGS_SECRET:-}"
|
||||
PG_HOST="${PG_HOST:-10.10.10.100}"
|
||||
PG_SUPERPASS="${PG_SUPERPASS:-}"
|
||||
SSH_KEY="${SSH_KEY:-/root/.ssh/hotel-manage_deploy}"
|
||||
|
||||
mkdir -p "$BACKUP_DIR/postgres/daily" "$BACKUP_DIR/postgres/weekly" "$BACKUP_DIR/volumes"
|
||||
log() { echo "[$(date -u '+%H:%M:%S')] $1"; }
|
||||
err() { log "ERROR: $1"; }
|
||||
|
||||
log() { echo "[$(date '+%H:%M:%S')] $1"; }
|
||||
RUN_STARTED=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
ITEMS_FILE=$(mktemp /tmp/backup_items.XXXXXX)
|
||||
echo '[]' > "$ITEMS_FILE"
|
||||
ERRORS=0
|
||||
|
||||
# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
|
||||
log "Starting PostgreSQL backups..."
|
||||
for DB in $BACKUP_DATABASES; do
|
||||
OUTFILE="$BACKUP_DIR/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
|
||||
PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" | gzip > "$OUTFILE"
|
||||
log " $DB → $(du -sh "$OUTFILE" | cut -f1)"
|
||||
add_item() {
|
||||
# args: type name status size error_msg
|
||||
local type="$1" name="$2" status="$3" size="${4:-0}" errmsg="${5:-}"
|
||||
local error_json='null'
|
||||
[ -n "$errmsg" ] && error_json="\"$(echo "$errmsg" | sed 's/"/\\"/g')\""
|
||||
jq --arg t "$type" --arg n "$name" --arg s "$status" \
|
||||
--argjson sz "$size" --argjson e "$error_json" \
|
||||
'. += [{"type":$t,"name":$n,"status":$s,"size":$sz,"error":$e}]' \
|
||||
"$ITEMS_FILE" > "${ITEMS_FILE}.tmp" && mv "${ITEMS_FILE}.tmp" "$ITEMS_FILE"
|
||||
}
|
||||
|
||||
finish() {
|
||||
local status="$1" error_msg="${2:-}"
|
||||
local run_finished=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
local items=$(cat "$ITEMS_FILE")
|
||||
local error_json='null'
|
||||
[ -n "$error_msg" ] && error_json="\"$(echo "$error_msg" | sed 's/"/\\"/g')\""
|
||||
rm -f "$ITEMS_FILE" "${ITEMS_FILE}.tmp"
|
||||
|
||||
printf '{"id":"%s","started":"%s","finished":"%s","status":"%s","items":%s,"error":%s}\n' \
|
||||
"$TIMESTAMP" "$RUN_STARTED" "$run_finished" "$status" "$items" "$error_json" \
|
||||
>> "$RUNS_FILE"
|
||||
|
||||
# Keep last 30 runs
|
||||
local lines=$(wc -l < "$RUNS_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$lines" -gt 30 ]; then
|
||||
tail -30 "$RUNS_FILE" > "${RUNS_FILE}.tmp" && mv "${RUNS_FILE}.tmp" "$RUNS_FILE"
|
||||
fi
|
||||
|
||||
# Uptime Kuma heartbeat
|
||||
if [ -n "$KUMA_PUSH_URL" ]; then
|
||||
local kuma_msg="Backup OK"
|
||||
[ "$status" = "partial" ] && kuma_msg="Backup partial (${ERRORS} errors)"
|
||||
[ "$status" = "error" ] && kuma_msg="Backup failed: ${error_msg}"
|
||||
wget -qO- "${KUMA_PUSH_URL}?status=up&msg=$(echo "$kuma_msg" | sed 's/ /+/g')&ping=" \
|
||||
> /dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
log "Done. Status: ${status}"
|
||||
}
|
||||
|
||||
# ── Fetch Nextcloud credentials ───────────────────────────────────────────────
|
||||
log "Fetching Nextcloud config from Settings..."
|
||||
NC_JSON=$(curl -sf --max-time 10 \
|
||||
-H "Authorization: Bearer ${SETTINGS_SECRET}" \
|
||||
"${SETTINGS_URL}/settings/api/internal/integration/nextcloud" 2>/dev/null) || NC_JSON=""
|
||||
|
||||
if [ -z "$NC_JSON" ]; then
|
||||
err "Could not reach Settings at ${SETTINGS_URL}"
|
||||
finish "error" "Settings service unreachable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NC_BASE_URL=$(echo "$NC_JSON" | jq -r '.base_url // empty' 2>/dev/null)
|
||||
NC_USER=$(echo "$NC_JSON" | jq -r '.username // empty' 2>/dev/null)
|
||||
NC_PASS=$(echo "$NC_JSON" | jq -r '.password // empty' 2>/dev/null)
|
||||
NC_PATH=$(echo "$NC_JSON" | jq -r '.backup_path // "HNF-Backups"' 2>/dev/null)
|
||||
|
||||
if [ -z "$NC_BASE_URL" ] || [ -z "$NC_USER" ] || [ -z "$NC_PASS" ]; then
|
||||
err "Nextcloud not configured in Settings (missing base_url, username, or password)"
|
||||
finish "error" "Nextcloud not configured"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NC_DAV="${NC_BASE_URL%/}/remote.php/dav/files/${NC_USER}/${NC_PATH}"
|
||||
log "Target: ${NC_BASE_URL} → /${NC_PATH}"
|
||||
|
||||
# Ensure Nextcloud directory tree exists (MKCOL silently ignores 405 = already exists)
|
||||
for dir in "" "/postgres" "/postgres/daily" "/postgres/weekly" "/volumes"; do
|
||||
curl -sf -X MKCOL -u "${NC_USER}:${NC_PASS}" \
|
||||
"${NC_DAV}${dir}" -o /dev/null 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Weekly full dump on Sundays
|
||||
if [ "$(date +%u)" = "7" ]; then
|
||||
log "Weekly full dump..."
|
||||
PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres \
|
||||
| gzip > "$BACKUP_DIR/postgres/weekly/full_${TIMESTAMP}.sql.gz"
|
||||
# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
|
||||
log "PostgreSQL backups..."
|
||||
for DB in $BACKUP_DATABASES; do
|
||||
log " ${DB}..."
|
||||
TARGET="${NC_DAV}/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
|
||||
# Write to temp file so we can check pg_dump exit separately from curl
|
||||
TMP_DUMP=$(mktemp /tmp/dump_${DB}.XXXXXX)
|
||||
if PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" 2>/dev/null \
|
||||
| gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
|
||||
HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
|
||||
-o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
|
||||
SIZE=$(wc -c < "$TMP_DUMP" 2>/dev/null || echo 0)
|
||||
if echo "$HTTP" | grep -qE '^2'; then
|
||||
log " OK (${SIZE} bytes compressed)"
|
||||
add_item "db" "$DB" "ok" "$SIZE"
|
||||
else
|
||||
err " ${DB} upload failed (HTTP ${HTTP})"
|
||||
add_item "db" "$DB" "error" "0" "Upload failed (HTTP ${HTTP})"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
err " ${DB} pg_dump failed"
|
||||
add_item "db" "$DB" "error" "0" "pg_dump failed"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
rm -f "$TMP_DUMP"
|
||||
done
|
||||
|
||||
# Weekly full pg_dumpall on Sundays
|
||||
if [ "$(date -u +%u)" = "7" ]; then
|
||||
log " Weekly pg_dumpall..."
|
||||
TARGET="${NC_DAV}/postgres/weekly/full_${TIMESTAMP}.sql.gz"
|
||||
TMP_DUMP=$(mktemp /tmp/dump_full.XXXXXX)
|
||||
if PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres 2>/dev/null \
|
||||
| gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
|
||||
HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
|
||||
-o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
|
||||
if echo "$HTTP" | grep -qE '^2'; then
|
||||
log " Weekly dump OK"
|
||||
else
|
||||
err " Weekly dump upload failed (HTTP ${HTTP})"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
err " pg_dumpall failed"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
rm -f "$TMP_DUMP"
|
||||
fi
|
||||
|
||||
# ── Retention cleanup ─────────────────────────────────────────────────────────
|
||||
find "$BACKUP_DIR/postgres/daily" -name "*.gz" -mtime +7 -delete
|
||||
find "$BACKUP_DIR/postgres/weekly" -name "*.gz" -mtime +28 -delete
|
||||
log "Retention cleanup done"
|
||||
# ── Volume backups via SSH ────────────────────────────────────────────────────
|
||||
# Each line: host:docker_volume_name:backup_label
|
||||
# Hosts that aren't deployed yet will fail silently (marked as skipped).
|
||||
log "Volume backups..."
|
||||
|
||||
# ── Rsync to remote ───────────────────────────────────────────────────────────
|
||||
if [ -n "$BACKUP_REMOTE" ]; then
|
||||
log "Syncing to $BACKUP_REMOTE..."
|
||||
rsync -az --delete \
|
||||
-e "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no" \
|
||||
"$BACKUP_DIR/" "$BACKUP_REMOTE"
|
||||
log "Sync complete"
|
||||
backup_volume() {
|
||||
local host="$1" volname="$2" label="$3"
|
||||
log " ${label} @ ${host}..."
|
||||
local target="${NC_DAV}/volumes/${label}_${TIMESTAMP}.tar.gz"
|
||||
local tmp=$(mktemp /tmp/vol_${label}.XXXXXX)
|
||||
|
||||
if ssh -i "$SSH_KEY" \
|
||||
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 -o BatchMode=yes \
|
||||
"root@${host}" \
|
||||
"docker run --rm --log-driver none -v ${volname}:/data alpine tar czf - -C /data . 2>/dev/null" \
|
||||
> "$tmp" 2>/dev/null && [ -s "$tmp" ]; then
|
||||
HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
|
||||
-o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
|
||||
SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
|
||||
if echo "$HTTP" | grep -qE '^2'; then
|
||||
log " ${label} OK (${SIZE} bytes)"
|
||||
add_item "vol" "$label" "ok" "$SIZE"
|
||||
else
|
||||
err " ${label} upload failed (HTTP ${HTTP})"
|
||||
add_item "vol" "$label" "error" "0" "Upload failed (HTTP ${HTTP})"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
log " ${label} skipped (host offline or volume empty)"
|
||||
add_item "vol" "$label" "skipped" "0"
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
backup_volume "10.10.10.117" "cashup_uploads_data" "cashup-uploads"
|
||||
backup_volume "10.10.10.121" "maintenance_uploads_data" "maintenance-uploads"
|
||||
backup_volume "10.10.10.113" "forecasting_recon_uploads" "forecasting-recon"
|
||||
backup_volume "10.10.10.110" "kitchen_invoice_data" "kitchen-invoices"
|
||||
|
||||
# Uptime Kuma data is mounted into this container at /kuma_data
|
||||
if [ -d /kuma_data ] && [ "$(ls -A /kuma_data 2>/dev/null)" ]; then
|
||||
log " kuma-data (local mount)..."
|
||||
target="${NC_DAV}/volumes/kuma-data_${TIMESTAMP}.tar.gz"
|
||||
tmp=$(mktemp /tmp/vol_kuma.XXXXXX)
|
||||
if tar czf "$tmp" -C /kuma_data . 2>/dev/null && [ -s "$tmp" ]; then
|
||||
HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
|
||||
-o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
|
||||
SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
|
||||
if echo "$HTTP" | grep -qE '^2'; then
|
||||
log " kuma-data OK (${SIZE} bytes)"
|
||||
add_item "vol" "kuma-data" "ok" "$SIZE"
|
||||
else
|
||||
err " kuma-data upload failed (HTTP ${HTTP})"
|
||||
add_item "vol" "kuma-data" "error" "0" "Upload failed (HTTP ${HTTP})"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
log " kuma-data skipped (empty)"
|
||||
add_item "vol" "kuma-data" "skipped" "0"
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
fi
|
||||
|
||||
# ── Uptime Kuma heartbeat ─────────────────────────────────────────────────────
|
||||
if [ -n "$KUMA_PUSH_URL" ]; then
|
||||
wget -qO- "$KUMA_PUSH_URL?status=up&msg=Backup+OK&ping=" > /dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
log "Backup complete"
|
||||
# ── Finalise ──────────────────────────────────────────────────────────────────
|
||||
STATUS="success"
|
||||
[ "$ERRORS" -gt 0 ] && STATUS="partial"
|
||||
finish "$STATUS"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue