homelab-upgrade/deploy/update.sh

#!/usr/bin/env bash
# Pulls the published image and restarts homelab-upgrade; rolls back if the new version does not come up.
# Lives next to compose.yaml on VM103 (/opt/homelab/compose/homelab-upgrade). See docs/DEPLOY.md.
#
#   ./update.sh              deploy the tag from .env (TAG=..., default "latest")
#   ./update.sh <tag>        deploy <tag> (a release tag, a branch name or a commit sha) and remember it in .env
#   ./update.sh --force      skip the "a workflow run is in the middle of a step" guard and a failed database copy
#   ./update.sh --install    first put this script's compose.yaml and update.sh into DEPLOY_DIR: the deploy workflow
#                            runs the copy inside the new image (/app/deploy) against the VM's directory
set -euo pipefail

script_dir="$(dirname "$(readlink -f "$0")")"
cd "${DEPLOY_DIR:-$script_dir}"

image_repo=git.mariuszpacek.pl/mariusz.pacek/homelab-upgrade
healthy_timeout=${HEALTHY_TIMEOUT:-90}
backups_kept=${BACKUPS_KEPT:-5}

tag_arg=""
force=0
install=0
for arg in "$@"; do
  case "$arg" in
    --force) force=1 ;;
    --install) install=1 ;;
    -*) echo "unknown option: $arg" >&2; exit 2 ;;
    *) tag_arg="$arg" ;;
  esac
done

if ((install == 1)) && [[ "$(pwd -P)" == "$(readlink -f "$script_dir")" ]]; then
  echo "--install needs DEPLOY_DIR set to another directory: a script cannot replace itself while it runs" >&2
  exit 2
fi

env_tag() {
  [[ -f .env ]] && sed -n 's/^TAG=//p' .env | tail -n 1 || true
}

service_container() {
  docker compose ps -q "$1" 2>/dev/null | head -n 1
}

# Exit 3 from the probe means the driver is in the middle of a node of a workflow run (a run waiting at a gate or queued
# between nodes survives a restart; a running node would be interrupted).
step_in_progress() {
  local rc=0
  docker compose exec -T server node -e "fetch('http://127.0.0.1:8080/api/workflow-instances').then((r) => r.json()).then((runs) => process.exit(runs.some((run) => run.status === 'running') ? 3 : 0), () => process.exit(4))" || rc=$?
  case "$rc" in
    0) return 1 ;;
    3) return 0 ;;
    *) echo "warning: could not ask the server about running workflow runs (rc=$rc); continuing" >&2; return 1 ;;
  esac
}

# A consistent online copy of the database through SQLite's backup API, taken by the running server (it owns /data);
# the newest $backups_kept copies stay.
backup_database() {
  docker compose exec -T server node -e "
const { DatabaseSync, backup } = require('node:sqlite');
const { readdirSync, rmSync } = require('node:fs');
const prefix = 'queue.sqlite.before-deploy-';
const name = prefix + new Date().toISOString().replace(/[-:]/g, '').replace(/[.][0-9]+Z\$/, 'Z');
const db = new DatabaseSync('/data/queue.sqlite', { readOnly: true });
backup(db, '/data/' + name).then(() => {
  db.close();
  for (const old of readdirSync('/data').filter((file) => file.startsWith(prefix)).sort().reverse().slice($backups_kept)) rmSync('/data/' + old);
  console.log('database copy: /data/' + name);
}, (error) => { console.error(error.message); process.exit(1); });
"
}

# The new compose.yaml and update.sh come from the image being deployed; the old compose file is kept, so a version
# that does not come up goes back together with the compose file it ran with.
install_files() {
  cp -p compose.yaml compose.yaml.previous
  install -m 600 "$script_dir/compose.yaml" compose.yaml
  install -m 755 "$script_dir/update.sh" update.sh
  if cmp -s compose.yaml.previous compose.yaml; then
    echo "compose.yaml unchanged"
  else
    echo "compose.yaml changed:"
    diff -u compose.yaml.previous compose.yaml || true
  fi
}

restore_compose() {
  if ((install == 1)) && [[ -f compose.yaml.previous ]]; then
    echo "restoring the previous compose.yaml" >&2
    cp -p compose.yaml.previous compose.yaml
  fi
}

wait_until_up() {
  local deadline=$((SECONDS + healthy_timeout)) status
  while ((SECONDS < deadline)); do
    status=$(docker inspect --format '{{.State.Health.Status}}' "$(service_container server)" 2>/dev/null || true)
    [[ "$status" == healthy ]] && break
    sleep 3
  done
  [[ "$status" == healthy ]] || return 1
  sleep 5 # the driver and the socket proxy have no healthcheck: they must still be running, and not restarting, after a few seconds
  local service
  for service in driver docker-proxy executor model-claude model-codex function-runtime; do
    [[ "$(docker inspect --format '{{.State.Status}} {{.RestartCount}}' "$(service_container "$service")" 2>/dev/null)" == "running 0" ]] || return 1
  done
  # The executor has a healthcheck of its own; give it the rest of the time to pass it.
  while ((SECONDS < deadline)); do
    [[ "$(docker inspect --format '{{.State.Health.Status}}' "$(service_container executor)" 2>/dev/null)" == healthy ]] && return 0
    sleep 3
  done
  return 1
}

tag="${tag_arg:-$(env_tag)}"
tag="${tag:-latest}"

if ((force == 0)) && [[ -n "$(service_container server)" ]] && step_in_progress; then
  echo "a workflow run is in the middle of a step; wait for it or use --force" >&2
  exit 3
fi

if [[ -n "$(service_container server)" ]] && backup_database; then
  :
elif ((force == 0)); then
  echo "could not copy the database (is the server running?); fix it or use --force" >&2
  exit 5
else
  echo "warning: no database copy (--force)" >&2
fi

previous_image=$(docker inspect --format '{{.Image}}' "$(service_container server)" 2>/dev/null || true)

((install == 1)) && install_files

echo "pulling $image_repo:$tag"
if ! TAG="$tag" docker compose pull; then
  echo "could not pull $image_repo:$tag; nothing was restarted" >&2
  restore_compose
  exit 1
fi
TAG="$tag" docker compose up -d --remove-orphans

if wait_until_up; then
  # Only the TAG line changes: .env also holds HA_TOKEN and other settings.
  if [[ -n "$tag_arg" ]]; then
    touch .env
    { grep -v '^TAG=' .env || true; printf 'TAG=%s\n' "$tag"; } > .env.new && chmod --reference=.env .env.new && mv .env.new .env
  fi
  revision=$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(service_container server)")
  echo "ok: $image_repo:$tag revision ${revision:-unknown}"
  exit 0
fi

echo "new version did not come up healthy" >&2
docker compose logs --tail 30 >&2 || true
restore_compose
if [[ -n "$previous_image" ]]; then
  echo "rolling back to $previous_image" >&2
  docker tag "$previous_image" "$image_repo:rollback"
  TAG=rollback docker compose up -d --remove-orphans
else
  echo "no previous image to roll back to" >&2
fi
exit 1