250 lines
8.8 KiB
Bash
Executable File
250 lines
8.8 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
cd "${1:-.}"
|
|
|
|
env_value_from() {
|
|
env_file="$1"
|
|
key="$2"
|
|
default_value="${3:-}"
|
|
value="$(grep -E "^${key}=" "$env_file" | tail -n 1 | cut -d= -f2- || true)"
|
|
if [ -n "$value" ]; then
|
|
printf '%s' "$value"
|
|
else
|
|
printf '%s' "$default_value"
|
|
fi
|
|
}
|
|
|
|
env_value() {
|
|
key="$1"
|
|
default_value="${2:-}"
|
|
env_value_from .env "$key" "$default_value"
|
|
}
|
|
|
|
cache_backend="$(env_value CRANK_CACHE_BACKEND memory)"
|
|
compose_profiles=""
|
|
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
|
compose_profiles="--profile cache"
|
|
fi
|
|
compose() {
|
|
# Intentional word splitting: compose_profiles is either empty or two arguments.
|
|
# shellcheck disable=SC2086
|
|
docker compose $compose_profiles "$@"
|
|
}
|
|
|
|
compose_up() {
|
|
postgres_scale=""
|
|
if [ "$(env_value POSTGRES_HOST postgres)" != "postgres" ] \
|
|
&& compose config --services | grep -Fxq postgres; then
|
|
postgres_scale="--scale postgres=0"
|
|
fi
|
|
# Intentional word splitting: postgres_scale is either empty or two arguments.
|
|
# shellcheck disable=SC2086
|
|
compose up -d --remove-orphans $postgres_scale
|
|
}
|
|
|
|
show_failure_diagnostics() {
|
|
compose ps >&2 || true
|
|
compose logs --no-color migrate >&2 || true
|
|
}
|
|
|
|
wait_for_stack() {
|
|
readiness_path="$1"
|
|
attempt=1
|
|
while [ "$attempt" -le 45 ]; do
|
|
if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
|
|
&& curl --fail --silent "http://127.0.0.1:3001/${readiness_path}" >/dev/null \
|
|
&& curl --fail --silent "http://127.0.0.1:3002/${readiness_path}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
find_artifact_container() {
|
|
expected_root="$1"
|
|
expected_mount="$(printf 'volume\t%s' "$expected_root")"
|
|
|
|
for service in admin-api artifact-storage-init; do
|
|
candidates="$(compose ps -aq "$service" 2>/dev/null || true)"
|
|
# Intentional word splitting: Docker container IDs cannot contain whitespace.
|
|
# shellcheck disable=SC2086
|
|
set -- $candidates
|
|
if [ "$#" -gt 1 ]; then
|
|
echo "Artifact backup found multiple $service containers; refusing an ambiguous volume source" >&2
|
|
return 1
|
|
fi
|
|
if [ "$#" -eq 1 ]; then
|
|
candidate="$1"
|
|
oneoff="$(docker inspect --format '{{index .Config.Labels "com.docker.compose.oneoff"}}' "$candidate" 2>/dev/null || true)"
|
|
mounts="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\n" .Type .Destination}}{{end}}' "$candidate" 2>/dev/null || true)"
|
|
if [ "$oneoff" != "True" ] && [ "$oneoff" != "true" ] \
|
|
&& printf '%s\n' "$mounts" | grep -Fxq "$expected_mount"; then
|
|
printf '%s' "$candidate"
|
|
return 0
|
|
fi
|
|
fi
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
create_backup() {
|
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
backup_dir="$(pwd)/backups/${timestamp}"
|
|
mkdir -p "$backup_dir"
|
|
chmod 700 "$backup_dir"
|
|
|
|
backup_env_file=.env
|
|
previous_deployment=false
|
|
if [ -f .env.previous ]; then
|
|
backup_env_file=.env.previous
|
|
previous_deployment=true
|
|
cp "$backup_env_file" "$backup_dir/runtime.env"
|
|
else
|
|
cp .env "$backup_dir/runtime.env"
|
|
fi
|
|
if [ -f docker-compose.previous.yml ]; then
|
|
cp docker-compose.previous.yml "$backup_dir/docker-compose.yml"
|
|
else
|
|
cp docker-compose.yml "$backup_dir/docker-compose.yml"
|
|
fi
|
|
|
|
postgres_host="$(env_value_from "$backup_env_file" POSTGRES_HOST)"
|
|
postgres_port="$(env_value_from "$backup_env_file" POSTGRES_PORT 5432)"
|
|
postgres_db="$(env_value_from "$backup_env_file" POSTGRES_DB crank)"
|
|
postgres_user="$(env_value_from "$backup_env_file" POSTGRES_USER crank)"
|
|
postgres_password="$(env_value_from "$backup_env_file" POSTGRES_PASSWORD)"
|
|
if [ -z "$postgres_host" ] || [ -z "$postgres_password" ]; then
|
|
echo "PostgreSQL credentials are required for the pre-update backup" >&2
|
|
return 1
|
|
fi
|
|
docker run --rm --network host \
|
|
-e PGPASSWORD="$postgres_password" \
|
|
-v "$backup_dir:/backup" \
|
|
postgres:16-alpine \
|
|
pg_dump --host "$postgres_host" --port "$postgres_port" \
|
|
--username "$postgres_user" --dbname "$postgres_db" \
|
|
--format custom --file /backup/postgres.dump
|
|
|
|
storage_root="$(env_value_from "$backup_env_file" CRANK_STORAGE_ROOT /var/lib/crank/storage)"
|
|
if artifact_container="$(find_artifact_container "$storage_root")"; then
|
|
docker run --rm --volumes-from "$artifact_container:ro" \
|
|
-v "$backup_dir:/backup" alpine:3.21 \
|
|
tar -C "$storage_root" -czf /backup/artifacts.tar.gz .
|
|
elif [ "$previous_deployment" = true ]; then
|
|
echo "Existing deployment container is unavailable; artifact backup cannot be verified" >&2
|
|
return 1
|
|
else
|
|
tar -czf "$backup_dir/artifacts.tar.gz" --files-from /dev/null
|
|
fi
|
|
|
|
(cd "$backup_dir" && sha256sum postgres.dump artifacts.tar.gz runtime.env docker-compose.yml > SHA256SUMS)
|
|
find backups -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \
|
|
| sort -nr | awk 'NR > 5 { print $2 }' | xargs -r rm -rf
|
|
}
|
|
|
|
validate_backup_migration() {
|
|
admin_image="$(env_value CRANK_ADMIN_API_IMAGE)"
|
|
if [ -z "$admin_image" ]; then
|
|
echo "CRANK_ADMIN_API_IMAGE is required for shadow migration validation" >&2
|
|
return 1
|
|
fi
|
|
|
|
shadow_id="crank-migration-check-$$"
|
|
shadow_network="${shadow_id}-network"
|
|
shadow_postgres="${shadow_id}-postgres"
|
|
shadow_password="crank-shadow-migration-password"
|
|
shadow_admin="crank_shadow_admin"
|
|
shadow_user="crank_shadow_owner"
|
|
shadow_cleanup() {
|
|
docker rm -fv "$shadow_postgres" >/dev/null 2>&1 || true
|
|
docker network rm "$shadow_network" >/dev/null 2>&1 || true
|
|
}
|
|
trap shadow_cleanup EXIT
|
|
trap 'exit 130' HUP INT TERM
|
|
|
|
docker network create "$shadow_network" >/dev/null
|
|
docker run -d --name "$shadow_postgres" --network "$shadow_network" \
|
|
-e POSTGRES_USER="$shadow_admin" \
|
|
-e POSTGRES_PASSWORD="$shadow_password" \
|
|
-e POSTGRES_DB=crank \
|
|
postgres:16-alpine >/dev/null
|
|
|
|
shadow_ready=false
|
|
attempt=1
|
|
while [ "$attempt" -le 30 ]; do
|
|
if docker logs "$shadow_postgres" 2>&1 | grep -q 'PostgreSQL init process complete' \
|
|
&& docker exec "$shadow_postgres" pg_isready --username "$shadow_admin" --dbname crank >/dev/null 2>&1; then
|
|
shadow_ready=true
|
|
break
|
|
fi
|
|
sleep 1
|
|
attempt=$((attempt + 1))
|
|
done
|
|
if [ "$shadow_ready" != true ]; then
|
|
echo "Shadow PostgreSQL did not become ready" >&2
|
|
return 1
|
|
fi
|
|
|
|
docker exec "$shadow_postgres" psql --username "$shadow_admin" --dbname crank \
|
|
--set ON_ERROR_STOP=1 \
|
|
--command "create role ${shadow_user} login password '${shadow_password}' nosuperuser nocreatedb nocreaterole noreplication" \
|
|
--command "alter database crank owner to ${shadow_user}"
|
|
|
|
timeout 10m docker run --rm --network "$shadow_network" \
|
|
-e PGPASSWORD="$shadow_password" \
|
|
-v "$backup_dir:/backup:ro" \
|
|
postgres:16-alpine \
|
|
pg_restore --host "$shadow_postgres" --username "$shadow_user" --dbname crank \
|
|
--no-owner --no-privileges --single-transaction --exit-on-error \
|
|
/backup/postgres.dump
|
|
|
|
shadow_database_url="postgres://${shadow_user}:${shadow_password}@${shadow_postgres}:5432/crank"
|
|
timeout 10m docker run --rm --network "$shadow_network" \
|
|
-e CRANK_DATABASE_URL="$shadow_database_url" \
|
|
"$admin_image" crank-migrate apply
|
|
shadow_preflight="$(timeout 2m docker run --rm --network "$shadow_network" \
|
|
-e CRANK_DATABASE_URL="$shadow_database_url" \
|
|
"$admin_image" crank-migrate preflight)"
|
|
printf '%s\n' "$shadow_preflight" | grep -Fxq '{"status":"current","version":13}'
|
|
|
|
shadow_cleanup
|
|
trap - EXIT HUP INT TERM
|
|
echo "Shadow migration validation passed"
|
|
}
|
|
|
|
rollback() {
|
|
echo "New release failed readiness; restoring previous deployment" >&2
|
|
if [ ! -f .env.previous ] || [ ! -f docker-compose.previous.yml ]; then
|
|
echo "Previous deployment metadata is unavailable" >&2
|
|
return 1
|
|
fi
|
|
cp .env .env.failed
|
|
cp docker-compose.yml docker-compose.failed.yml
|
|
cp .env.previous .env
|
|
cp docker-compose.previous.yml docker-compose.yml
|
|
cache_backend="$(env_value CRANK_CACHE_BACKEND memory)"
|
|
compose_profiles=""
|
|
if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then
|
|
compose_profiles="--profile cache"
|
|
fi
|
|
compose_up
|
|
wait_for_stack health
|
|
}
|
|
|
|
compose config -q
|
|
create_backup
|
|
compose pull
|
|
validate_backup_migration
|
|
|
|
if ! compose_up || ! wait_for_stack ready; then
|
|
show_failure_diagnostics
|
|
rollback
|
|
exit 1
|
|
fi
|
|
|
|
compose ps
|