management/backup/backup.sh

#!/bin/sh
set -e

TIMESTAMP=$(date -u '+%Y%m%d_%H%M%S')
RUNS_FILE=/backups/runs.ndjson
SETTINGS_URL="${SETTINGS_URL:-http://10.10.10.116:3080}"
SETTINGS_SECRET="${SETTINGS_SECRET:-}"
PG_HOST="${PG_HOST:-10.10.10.100}"
PG_SUPERPASS="${PG_SUPERPASS:-}"
SSH_KEY="${SSH_KEY:-/root/.ssh/hotel-manage_deploy}"

log()  { echo "[$(date -u '+%H:%M:%S')] $1"; }
err()  { log "ERROR: $1"; }

RUN_STARTED=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
ITEMS_FILE=$(mktemp /tmp/backup_items.XXXXXX)
echo '[]' > "$ITEMS_FILE"
ERRORS=0

add_item() {
  # args: type name status size error_msg
  local type="$1" name="$2" status="$3" size="${4:-0}" errmsg="${5:-}"
  local error_json='null'
  [ -n "$errmsg" ] && error_json="\"$(echo "$errmsg" | sed 's/"/\\"/g')\""
  jq --arg t "$type" --arg n "$name" --arg s "$status" \
     --argjson sz "$size" --argjson e "$error_json" \
     '. += [{"type":$t,"name":$n,"status":$s,"size":$sz,"error":$e}]' \
     "$ITEMS_FILE" > "${ITEMS_FILE}.tmp" && mv "${ITEMS_FILE}.tmp" "$ITEMS_FILE"
}

finish() {
  local status="$1" error_msg="${2:-}"
  local run_finished=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
  local items=$(jq -c '.' "$ITEMS_FILE")
  local error_json='null'
  [ -n "$error_msg" ] && error_json="\"$(echo "$error_msg" | sed 's/"/\\"/g')\""
  rm -f "$ITEMS_FILE" "${ITEMS_FILE}.tmp"

  printf '{"id":"%s","started":"%s","finished":"%s","status":"%s","items":%s,"error":%s}\n' \
    "$TIMESTAMP" "$RUN_STARTED" "$run_finished" "$status" "$items" "$error_json" \
    >> "$RUNS_FILE"

  # Keep last 30 runs
  local lines=$(wc -l < "$RUNS_FILE" 2>/dev/null || echo 0)
  if [ "$lines" -gt 30 ]; then
    tail -30 "$RUNS_FILE" > "${RUNS_FILE}.tmp" && mv "${RUNS_FILE}.tmp" "$RUNS_FILE"
  fi

  # Uptime Kuma heartbeat
  if [ -n "$KUMA_PUSH_URL" ]; then
    local kuma_msg="Backup OK"
    [ "$status" = "partial" ] && kuma_msg="Backup partial (${ERRORS} errors)"
    [ "$status" = "error" ]   && kuma_msg="Backup failed: ${error_msg}"
    wget -qO- "${KUMA_PUSH_URL}?status=up&msg=$(echo "$kuma_msg" | sed 's/ /+/g')&ping=" \
      > /dev/null 2>&1 || true
  fi

  log "Done. Status: ${status}"
}

# ── Fetch Nextcloud credentials ───────────────────────────────────────────────
log "Fetching Nextcloud config from Settings..."
NC_JSON=$(curl -sf --max-time 10 \
  -H "Authorization: Bearer ${SETTINGS_SECRET}" \
  "${SETTINGS_URL}/settings/api/internal/integration/nextcloud" 2>/dev/null) || NC_JSON=""

if [ -z "$NC_JSON" ]; then
  err "Could not reach Settings at ${SETTINGS_URL}"
  finish "error" "Settings service unreachable"
  exit 1
fi

NC_BASE_URL=$(echo "$NC_JSON" | jq -r '.base_url // empty' 2>/dev/null)
NC_USER=$(echo     "$NC_JSON" | jq -r '.username // empty'  2>/dev/null)
NC_PASS=$(echo     "$NC_JSON" | jq -r '.password // empty'  2>/dev/null)
NC_PATH=$(echo     "$NC_JSON" | jq -r '.backup_path // "HNF-Backups"' 2>/dev/null)

if [ -z "$NC_BASE_URL" ] || [ -z "$NC_USER" ] || [ -z "$NC_PASS" ]; then
  err "Nextcloud not configured in Settings (missing base_url, username, or password)"
  finish "error" "Nextcloud not configured"
  exit 1
fi

NC_DAV="${NC_BASE_URL%/}/remote.php/dav/files/${NC_USER}/${NC_PATH}"
log "Target: ${NC_BASE_URL} → /${NC_PATH}"

# Ensure Nextcloud directory tree exists (MKCOL silently ignores 405 = already exists)
for dir in "" "/postgres" "/postgres/daily" "/postgres/weekly" "/volumes"; do
  curl -sf -X MKCOL -u "${NC_USER}:${NC_PASS}" \
    "${NC_DAV}${dir}" -o /dev/null 2>/dev/null || true
done

# ── PostgreSQL dumps ──────────────────────────────────────────────────────────
log "PostgreSQL backups..."
for DB in $BACKUP_DATABASES; do
  log "  ${DB}..."
  TARGET="${NC_DAV}/postgres/daily/${DB}_${TIMESTAMP}.sql.gz"
  # Write to temp file so we can check pg_dump exit separately from curl
  TMP_DUMP=$(mktemp /tmp/dump_${DB}.XXXXXX)
  if PGPASSWORD="$PG_SUPERPASS" pg_dump -h "$PG_HOST" -U postgres "$DB" 2>/dev/null \
       | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
    HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
           -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
    SIZE=$(wc -c < "$TMP_DUMP" 2>/dev/null || echo 0)
    if echo "$HTTP" | grep -qE '^2'; then
      log "    OK (${SIZE} bytes compressed)"
      add_item "db" "$DB" "ok" "$SIZE"
    else
      err "  ${DB} upload failed (HTTP ${HTTP})"
      add_item "db" "$DB" "error" "0" "Upload failed (HTTP ${HTTP})"
      ERRORS=$((ERRORS + 1))
    fi
  else
    err "  ${DB} pg_dump failed"
    add_item "db" "$DB" "error" "0" "pg_dump failed"
    ERRORS=$((ERRORS + 1))
  fi
  rm -f "$TMP_DUMP"
done

# Weekly full pg_dumpall on Sundays
if [ "$(date -u +%u)" = "7" ]; then
  log "  Weekly pg_dumpall..."
  TARGET="${NC_DAV}/postgres/weekly/full_${TIMESTAMP}.sql.gz"
  TMP_DUMP=$(mktemp /tmp/dump_full.XXXXXX)
  if PGPASSWORD="$PG_SUPERPASS" pg_dumpall -h "$PG_HOST" -U postgres 2>/dev/null \
       | gzip > "$TMP_DUMP" && [ -s "$TMP_DUMP" ]; then
    HTTP=$(curl -sf -T "$TMP_DUMP" -u "${NC_USER}:${NC_PASS}" \
           -o /dev/null -w "%{http_code}" "$TARGET" 2>/dev/null || echo "000")
    if echo "$HTTP" | grep -qE '^2'; then
      log "    Weekly dump OK"
    else
      err "  Weekly dump upload failed (HTTP ${HTTP})"
      ERRORS=$((ERRORS + 1))
    fi
  else
    err "  pg_dumpall failed"
    ERRORS=$((ERRORS + 1))
  fi
  rm -f "$TMP_DUMP"
fi

# ── Volume backups via SSH ────────────────────────────────────────────────────
# Each line: host:docker_volume_name:backup_label
# Hosts that aren't deployed yet will fail silently (marked as skipped).
log "Volume backups..."

backup_volume() {
  local host="$1" volname="$2" label="$3"
  log "  ${label} @ ${host}..."
  local target="${NC_DAV}/volumes/${label}_${TIMESTAMP}.tar.gz"
  local tmp=$(mktemp /tmp/vol_${label}.XXXXXX)

  if ssh -i "$SSH_KEY" \
        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
        -o ConnectTimeout=5 -o BatchMode=yes \
        "root@${host}" \
        "docker run --rm --log-driver none -v ${volname}:/data alpine tar czf - -C /data . 2>/dev/null" \
        > "$tmp" 2>/dev/null && [ -s "$tmp" ]; then
    HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
           -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
    SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
    if echo "$HTTP" | grep -qE '^2'; then
      log "    ${label} OK (${SIZE} bytes)"
      add_item "vol" "$label" "ok" "$SIZE"
    else
      err "  ${label} upload failed (HTTP ${HTTP})"
      add_item "vol" "$label" "error" "0" "Upload failed (HTTP ${HTTP})"
      ERRORS=$((ERRORS + 1))
    fi
  else
    log "    ${label} skipped (host offline or volume empty)"
    add_item "vol" "$label" "skipped" "0"
  fi
  rm -f "$tmp"
}

backup_volume "10.10.10.117" "cashup_uploads_data"           "cashup-uploads"
backup_volume "10.10.10.121" "maintenance_uploads_data"      "maintenance-uploads"
backup_volume "10.10.10.113" "forecasting_recon_uploads"     "forecasting-recon"
backup_volume "10.10.10.110" "kitchen_invoice_data"          "kitchen-invoices"

# Uptime Kuma data is mounted into this container at /kuma_data
if [ -d /kuma_data ] && [ "$(ls -A /kuma_data 2>/dev/null)" ]; then
  log "  kuma-data (local mount)..."
  target="${NC_DAV}/volumes/kuma-data_${TIMESTAMP}.tar.gz"
  tmp=$(mktemp /tmp/vol_kuma.XXXXXX)
  if tar czf "$tmp" -C /kuma_data . 2>/dev/null && [ -s "$tmp" ]; then
    HTTP=$(curl -sf -T "$tmp" -u "${NC_USER}:${NC_PASS}" \
           -o /dev/null -w "%{http_code}" "$target" 2>/dev/null || echo "000")
    SIZE=$(wc -c < "$tmp" 2>/dev/null || echo 0)
    if echo "$HTTP" | grep -qE '^2'; then
      log "    kuma-data OK (${SIZE} bytes)"
      add_item "vol" "kuma-data" "ok" "$SIZE"
    else
      err "  kuma-data upload failed (HTTP ${HTTP})"
      add_item "vol" "kuma-data" "error" "0" "Upload failed (HTTP ${HTTP})"
      ERRORS=$((ERRORS + 1))
    fi
  else
    log "    kuma-data skipped (empty)"
    add_item "vol" "kuma-data" "skipped" "0"
  fi
  rm -f "$tmp"
fi

# ── Finalise ──────────────────────────────────────────────────────────────────
STATUS="success"
[ "$ERRORS" -gt 0 ] && STATUS="partial"
finish "$STATUS"