jtricerolph
377a7478f7
Collects .env files from all app LXCs via SSH and uploads a single env_bundle_TIMESTAMP.tar.gz to HNF-Backups/configs/ on Nextcloud. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
270 lines
11 KiB
Bash
270 lines
11 KiB
Bash
#!/bin/sh
|
|
set -e
|
|
|
|
TIMESTAMP=$(date -u '+%Y%m%d_%H%M%S')
|
|
RUNS_FILE=/backups/runs.ndjson
|
|
SETTINGS_URL="${SETTINGS_URL:-http://10.10.10.116:3080}"
|
|
SETTINGS_SECRET="${SETTINGS_SECRET:-}"
|
|
PG_HOST="${PG_HOST:-10.10.10.100}"
|
|
PG_SUPERPASS="${PG_SUPERPASS:-}"
|
|
SSH_KEY="${SSH_KEY:-/root/.ssh/hotel-manage_deploy}"
|
|
|
|
log() { echo "[$(date -u '+%H:%M:%S')] $1"; }
|
|
err() { log "ERROR: $1"; }
|
|
|
|
RUN_STARTED=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
ITEMS_FILE=$(mktemp /tmp/backup_items.XXXXXX)
|
|
echo '[]' > "$ITEMS_FILE"
|
|
ERRORS=0
|
|
|
|
add_item() {
|
|
# args: type name status size error_msg
|
|
local type="$1" name="$2" status="$3" size="${4:-0}" errmsg="${5:-}"
|
|
local error_json='null'
|
|
[ -n "$errmsg" ] && error_json="\"$(echo "$errmsg" | sed 's/"/\\"/g')\""
|
|
jq --arg t "$type" --arg n "$name" --arg s "$status" \
|
|
--argjson sz "$size" --argjson e "$error_json" \
|
|
'. += [{"type":$t,"name":$n,"status":$s,"size":$sz,"error":$e}]' \
|
|
"$ITEMS_FILE" > "${ITEMS_FILE}.tmp" && mv "${ITEMS_FILE}.tmp" "$ITEMS_FILE"
|
|
}
|
|
|
|
finish() {
|
|
local status="$1" error_msg="${2:-}"
|
|
local run_finished=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
local items=$(jq -c '.' "$ITEMS_FILE")
|
|
local error_json='null'
|
|
[ -n "$error_msg" ] && error_json="\"$(echo "$error_msg" | sed 's/"/\\"/g')\""
|
|
rm -f "$ITEMS_FILE" "${ITEMS_FILE}.tmp"
|
|
|
|
printf '{"id":"%s","started":"%s","finished":"%s","status":"%s","items":%s,"error":%s}\n' \
|
|
"$TIMESTAMP" "$RUN_STARTED" "$run_finished" "$status" "$items" "$error_json" \
|
|
>> "$RUNS_FILE"
|
|
|
|
# Keep last 30 runs
|
|
local lines=$(wc -l < "$RUNS_FILE" 2>/dev/null || echo 0)
|
|
if [ "$lines" -gt 30 ]; then
|
|
tail -30 "$RUNS_FILE" > "${RUNS_FILE}.tmp" && mv "${RUNS_FILE}.tmp" "$RUNS_FILE"
|
|
fi
|
|
|
|
# Uptime Kuma heartbeat
|
|
if [ -n "$KUMA_PUSH_URL" ]; then
|
|
local kuma_msg="Backup OK"
|
|
[ "$status" = "partial" ] && kuma_msg="Backup partial (${ERRORS} errors)"
|
|
[ "$status" = "error" ] && kuma_msg="Backup failed: ${error_msg}"
|
|
wget -qO- "${KUMA_PUSH_URL}?status=up&msg=$(echo "$kuma_msg" | sed 's/ /+/g')&ping=" \
|
|
> /dev/null 2>&1 || true
|
|
fi
|
|
|
|
log "Done. Status: ${status}"
|
|
}
|
|
|
|
# ── Fetch Nextcloud credentials ───────────────────────────────────────────────
|
|
log "Fetching Nextcloud config from Settings..."
|
|
NC_JSON=$(curl -sf --max-time 10 \
|
|
-H "Authorization: Bearer ${SETTINGS_SECRET}" \
|
|
"${SETTINGS_URL}/settings/api/internal/integration/nextcloud" 2>/dev/null) || NC_JSON=""
|
|
|
|
if [ -z "$NC_JSON" ]; then
|
|
err "Could not reach Settings at ${SETTINGS_URL}"
|
|
finish "error" "Settings service unreachable"
|
|
exit 1
|
|
fi
|
|
|
|
NC_BASE_URL=$(echo "$NC_JSON" | jq -r '.base_url // empty' 2>/dev/null)
|
|
NC_USER=$(echo "$NC_JSON" | jq -r '.username // empty' 2>/dev/null)
|
|
NC_PASS=$(echo "$NC_JSON" | jq -r '.password // empty' 2>/dev/null)
|
|
NC_PATH=$(echo "$NC_JSON" | jq -r '.backup_path // "HNF-Backups"' 2>/dev/null)
|
|
|
|
if [ -z "$NC_BASE_URL" ] || [ -z "$NC_USER" ] || [ -z "$NC_PASS" ]; then
|
|
err "Nextcloud not configured in Settings (missing base_url, username, or password)"
|
|
finish "error" "Nextcloud not configured"
|
|
exit 1
|
|
fi
|
|
|
|
NC_DAV="${NC_BASE_URL%/}/remote.php/dav/files/${NC_USER}/${NC_PATH}"
|
|
log "Target: ${NC_BASE_URL} → /${NC_PATH}"
|
|
|
|
# Ensure Nextcloud directory tree exists (MKCOL silently ignores 405 = already exists)
|
|
for dir in "" "/postgres" "/postgres/daily" "/postgres/weekly" "/volumes" "/configs"; do
|
|
curl -sf -X MKCOL -u "${NC_USER}:${NC_PASS}" \
|
|
"${NC_DAV}${dir}" -o /dev/null 2>/dev/null || true
|
|
done
|
|
|
|
# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
|
|
log "PostgreSQL backups..."
|
|
BACKUP_DATABASES=$(PGPASSWORD="$PG_SUPERPASS" psql -h "$PG_HOST" -U postgres -t -A -c \
|
|
"SELECT datname FROM pg_database WHERE datistemplate = false AND datname != 'postgres' ORDER BY datname" \
|
|
2>/dev/null) || true
|
|
if [ -z "$BACKUP_DATABASES" ]; then
|
|
err "Could not list databases from postgres"
|
|
finish "error" "Could not connect to postgres to list databases"
|
|
exit 1
|
|
fi
|
|
for DB in $BACKUP_DATABASES; do
|
|
log " ${DB}..."
|
|
TARGET="${NC_DAV}/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
|
|
# Write to temp file so we can check pg_dump exit separately from curl
|
|
TMP_DUMP=$(mktemp /tmp/dump_${DB}.XXXXXX)
|
|
if PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" 2>/dev/null \
|
|
| gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
|
|
HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
|
|
-o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
|
|
SIZE=$(wc -c < "$TMP_DUMP" 2>/dev/null || echo 0)
|
|
if echo "$HTTP" | grep -qE '^2'; then
|
|
log " OK (${SIZE} bytes compressed)"
|
|
add_item "db" "$DB" "ok" "$SIZE"
|
|
else
|
|
err " ${DB} upload failed (HTTP ${HTTP})"
|
|
add_item "db" "$DB" "error" "0" "Upload failed (HTTP ${HTTP})"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
else
|
|
err " ${DB} pg_dump failed"
|
|
add_item "db" "$DB" "error" "0" "pg_dump failed"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
rm -f "$TMP_DUMP"
|
|
done
|
|
|
|
# Weekly full pg_dumpall on Sundays
|
|
if [ "$(date -u +%u)" = "7" ]; then
|
|
log " Weekly pg_dumpall..."
|
|
TARGET="${NC_DAV}/postgres/weekly/full_${TIMESTAMP}.sql.gz"
|
|
TMP_DUMP=$(mktemp /tmp/dump_full.XXXXXX)
|
|
if PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres 2>/dev/null \
|
|
| gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
|
|
HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
|
|
-o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
|
|
if echo "$HTTP" | grep -qE '^2'; then
|
|
log " Weekly dump OK"
|
|
else
|
|
err " Weekly dump upload failed (HTTP ${HTTP})"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
else
|
|
err " pg_dumpall failed"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
rm -f "$TMP_DUMP"
|
|
fi
|
|
|
|
# ── Volume backups via SSH ────────────────────────────────────────────────────
|
|
# Each line: host:docker_volume_name:backup_label
|
|
# Hosts that aren't deployed yet will fail silently (marked as skipped).
|
|
log "Volume backups..."
|
|
|
|
backup_volume() {
|
|
local host="$1" volname="$2" label="$3"
|
|
log " ${label} @ ${host}..."
|
|
local target="${NC_DAV}/volumes/${label}_${TIMESTAMP}.tar.gz"
|
|
local tmp=$(mktemp /tmp/vol_${label}.XXXXXX)
|
|
|
|
if ssh -i "$SSH_KEY" \
|
|
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=5 -o BatchMode=yes \
|
|
"root@${host}" \
|
|
"docker run --rm --log-driver none -v ${volname}:/data alpine tar czf - -C /data . 2>/dev/null" \
|
|
> "$tmp" 2>/dev/null && [ -s "$tmp" ]; then
|
|
HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
|
|
-o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
|
|
SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
|
|
if echo "$HTTP" | grep -qE '^2'; then
|
|
log " ${label} OK (${SIZE} bytes)"
|
|
add_item "vol" "$label" "ok" "$SIZE"
|
|
else
|
|
err " ${label} upload failed (HTTP ${HTTP})"
|
|
add_item "vol" "$label" "error" "0" "Upload failed (HTTP ${HTTP})"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
else
|
|
log " ${label} skipped (host offline or volume empty)"
|
|
add_item "vol" "$label" "skipped" "0"
|
|
fi
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
backup_volume "10.10.10.117" "cashup_uploads_data" "cashup-uploads"
|
|
backup_volume "10.10.10.121" "maintenance_uploads_data" "maintenance-uploads"
|
|
backup_volume "10.10.10.113" "forecasting_recon_uploads" "forecasting-recon"
|
|
backup_volume "10.10.10.110" "kitchen_invoice_data" "kitchen-invoices"
|
|
|
|
# Uptime Kuma data is mounted into this container at /kuma_data
|
|
if [ -d /kuma_data ] && [ "$(ls -A /kuma_data 2>/dev/null)" ]; then
|
|
log " kuma-data (local mount)..."
|
|
target="${NC_DAV}/volumes/kuma-data_${TIMESTAMP}.tar.gz"
|
|
tmp=$(mktemp /tmp/vol_kuma.XXXXXX)
|
|
if tar czf "$tmp" -C /kuma_data . 2>/dev/null && [ -s "$tmp" ]; then
|
|
HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
|
|
-o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
|
|
SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
|
|
if echo "$HTTP" | grep -qE '^2'; then
|
|
log " kuma-data OK (${SIZE} bytes)"
|
|
add_item "vol" "kuma-data" "ok" "$SIZE"
|
|
else
|
|
err " kuma-data upload failed (HTTP ${HTTP})"
|
|
add_item "vol" "kuma-data" "error" "0" "Upload failed (HTTP ${HTTP})"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
else
|
|
log " kuma-data skipped (empty)"
|
|
add_item "vol" "kuma-data" "skipped" "0"
|
|
fi
|
|
rm -f "$tmp"
|
|
fi
|
|
|
|
# ── Config (.env) backups ─────────────────────────────────────────────────────
|
|
log "Config (.env) backups..."
|
|
TMP_ENV_DIR=$(mktemp -d /tmp/envs.XXXXXX)
|
|
ENV_COLLECTED=0
|
|
|
|
collect_env() {
|
|
local host="$1" app="$2"
|
|
if ssh -i "$SSH_KEY" \
|
|
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=5 -o BatchMode=yes \
|
|
"root@${host}" "cat /opt/${app}/.env" \
|
|
> "${TMP_ENV_DIR}/${app}.env" 2>/dev/null \
|
|
&& [ -s "${TMP_ENV_DIR}/${app}.env" ]; then
|
|
log " ${app} collected"
|
|
ENV_COLLECTED=$((ENV_COLLECTED + 1))
|
|
else
|
|
log " ${app} skipped (offline or no .env)"
|
|
rm -f "${TMP_ENV_DIR}/${app}.env"
|
|
fi
|
|
}
|
|
|
|
collect_env "10.10.10.101" "auth"
|
|
collect_env "10.10.10.105" "management"
|
|
collect_env "10.10.10.112" "noticeboard"
|
|
collect_env "10.10.10.113" "forecasting"
|
|
collect_env "10.10.10.115" "rates"
|
|
collect_env "10.10.10.116" "settings"
|
|
collect_env "10.10.10.117" "cashup"
|
|
collect_env "10.10.10.118" "hk-planner"
|
|
collect_env "10.10.10.119" "twin-optimiser"
|
|
|
|
TMP_ENV_BUNDLE=$(mktemp /tmp/envs_bundle.XXXXXX)
|
|
if [ "$ENV_COLLECTED" -gt 0 ] && tar czf "$TMP_ENV_BUNDLE" -C "$TMP_ENV_DIR" . 2>/dev/null && [ -s "$TMP_ENV_BUNDLE" ]; then
|
|
TARGET="${NC_DAV}/configs/env_bundle_${TIMESTAMP}.tar.gz"
|
|
HTTP=$(curl -sf -T "$TMP_ENV_BUNDLE" -u "${NC_USER}:${NC_PASS}" \
|
|
-o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
|
|
SIZE=$(wc -c < "$TMP_ENV_BUNDLE" 2>/dev/null || echo 0)
|
|
if echo "$HTTP" | grep -qE '^2'; then
|
|
log " env bundle OK (${ENV_COLLECTED} files, ${SIZE} bytes)"
|
|
add_item "config" "env-bundle" "ok" "$SIZE"
|
|
else
|
|
err " env bundle upload failed (HTTP ${HTTP})"
|
|
add_item "config" "env-bundle" "error" "0" "Upload failed (HTTP ${HTTP})"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
else
|
|
err " env bundle: nothing collected"
|
|
add_item "config" "env-bundle" "error" "0" "No .env files collected"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
rm -rf "$TMP_ENV_DIR" "$TMP_ENV_BUNDLE"
|
|
|
|
# ── Finalise ──────────────────────────────────────────────────────────────────
|
|
STATUS="success"
|
|
[ "$ERRORS" -gt 0 ] && STATUS="partial"
|
|
finish "$STATUS"
|