Harden self-update with health-check and automatic rollback

The updater now snapshots the current commit before touching anything. If the
build fails, or the API does not answer /health within ~60s after the PM2 reload,
the checkout is reset back to that commit, rebuilt and reloaded — so a failed
update can no longer leave the instance down. Reads API_PORT from .env to probe
http://127.0.0.1:PORT/health (curl, then wget, then a raw TCP fallback).

Author: softpython2884

scripts/self-update.sh

@@ -4,6 +4,10 @@
 # detached, so it survives the PM2 reload it performs at the end). It resets the checkout to
 # the tracked branch, rebuilds/migrates via deploy.sh, and reloads the PM2 processes.
 #
+# SAFETY (hardened): the current commit is snapshotted *before* anything changes. If the build
+# fails, OR the API does not answer /health after the reload, the checkout is rolled back to that
+# snapshot, rebuilt and reloaded — so a bad update can no longer leave the instance down.
+#
 # Progress is written to .update-status.json (read by the API) and full output to .update.log.
 #
 # NOTE: this discards local changes on the tracked branch with `git reset --hard`. Operators
@@ -24,31 +28,88 @@ write_status() {
     "$state" "$STARTED" "$finished" "$message" > "$STATUS"
 }
 
-fail() {
-  echo "FAILED: $1" >> "$LOG"
-  write_status "failed" "$1" "\"$(date -Iseconds)\""
+# Read a single value from .env WITHOUT sourcing it (mirrors deploy.sh).
+read_env() {
+  local key="$1" line val
+  [[ -f .env ]] || { printf ''; return; }
+  line="$(grep -E "^${key}=" .env | tail -1 || true)"
+  val="${line#*=}"
+  [[ "$val" == \"*\" ]] && val="${val:1:${#val}-2}"
+  [[ "$val" == \'*\' ]] && val="${val:1:${#val}-2}"
+  printf '%s' "$val"
+}
+
+API_PORT="$(read_env API_PORT)"; : "${API_PORT:=4000}"
+HEALTH_URL="http://127.0.0.1:${API_PORT}/health"
+
+reload_pm2() {
+  if command -v pm2 >/dev/null 2>&1; then
+    { pm2 reload "$ROOT/ecosystem.config.cjs"; } >> "$LOG" 2>&1 || echo "pm2 reload warning" >> "$LOG"
+  else
+    echo "pm2 not found on PATH — reload skipped; restart the processes manually." >> "$LOG"
+  fi
+}
+
+# True once the API answers its /health endpoint. Tries curl, then wget, then a raw TCP probe.
+http_ok() {
+  if command -v curl >/dev/null 2>&1; then
+    curl -fsS --max-time 4 "$HEALTH_URL" >/dev/null 2>&1 && return 0
+    return 1
+  fi
+  if command -v wget >/dev/null 2>&1; then
+    wget -q -T 4 -O /dev/null "$HEALTH_URL" >/dev/null 2>&1 && return 0
+    return 1
+  fi
+  # Last resort: just check the port accepts a connection.
+  (exec 3<>"/dev/tcp/127.0.0.1/${API_PORT}") 2>/dev/null && return 0
+  return 1
+}
+
+# Poll /health for up to ~60s (30 × 2s). PM2 reload is graceful, so give the new process time.
+wait_healthy() {
+  local i
+  for i in $(seq 1 30); do
+    if http_ok; then return 0; fi
+    sleep 2
+  done
+  return 1
+}
+
+# Roll the checkout back to the pre-update commit and bring it back up. Best-effort.
+rollback() {
+  local reason="$1"
+  echo "ROLLBACK ($reason) → $PREV_SHA" >> "$LOG"
+  write_status "running" "Échec — restauration de la version précédente…" "null"
+  { git reset --hard "$PREV_SHA"; } >> "$LOG" 2>&1
+  { ./scripts/deploy.sh; } >> "$LOG" 2>&1 || echo "rollback rebuild warning" >> "$LOG"
+  reload_pm2
+  write_status "failed" "Mise à jour annulée et version précédente restaurée ($reason)." "\"$(date -Iseconds)\""
+  echo "== rolled back $(date -Iseconds) ==" >> "$LOG"
   exit 1
 }
 
 : > "$LOG"
 echo "== self-update $(date -Iseconds) (branch $BRANCH) ==" >> "$LOG"
 write_status "running" "Récupération des sources…" "null"
 
-{ git fetch --all --prune; } >> "$LOG" 2>&1 || fail "git fetch a échoué"
-{ git checkout "$BRANCH"; } >> "$LOG" 2>&1 || fail "git checkout $BRANCH a échoué"
-{ git reset --hard "origin/$BRANCH"; } >> "$LOG" 2>&1 || fail "git reset a échoué"
+# Snapshot the current commit so we can return to it if anything goes wrong.
+PREV_SHA="$(git rev-parse HEAD 2>/dev/null || echo '')"
+[[ -n "$PREV_SHA" ]] || { write_status "failed" "Impossible de lire le commit courant (pas un dépôt git ?)." "\"$(date -Iseconds)\""; exit 1; }
+echo "snapshot PREV_SHA=$PREV_SHA" >> "$LOG"
 
-write_status "running" "Build et migrations…" "null"
-{ ./scripts/deploy.sh; } >> "$LOG" 2>&1 || fail "le build/déploiement a échoué"
+{ git fetch --all --prune; } >> "$LOG" 2>&1 || { write_status "failed" "git fetch a échoué." "\"$(date -Iseconds)\""; exit 1; }
+{ git checkout "$BRANCH"; } >> "$LOG" 2>&1 || { write_status "failed" "git checkout $BRANCH a échoué." "\"$(date -Iseconds)\""; exit 1; }
+{ git reset --hard "origin/$BRANCH"; } >> "$LOG" 2>&1 || rollback "git reset a échoué"
 
-# Mark success before reloading: the reload restarts this very API process, which then reads
-# the status file. The reload itself is best-effort.
-write_status "success" "Mise à jour appliquée, redémarrage…" "\"$(date -Iseconds)\""
+write_status "running" "Build et migrations…" "null"
+{ ./scripts/deploy.sh; } >> "$LOG" 2>&1 || rollback "le build/déploiement a échoué"
 
-if command -v pm2 >/dev/null 2>&1; then
-  { pm2 reload "$ROOT/ecosystem.config.cjs"; } >> "$LOG" 2>&1 || echo "pm2 reload warning" >> "$LOG"
-else
-  echo "pm2 not found on PATH — reload skipped; restart the processes manually." >> "$LOG"
+# Reload, then verify the API actually comes back before declaring success.
+write_status "running" "Redémarrage et vérification de l'état…" "null"
+reload_pm2
+if ! wait_healthy; then
+  rollback "l'API ne répond plus après le redémarrage"
 fi
 
+write_status "success" "Mise à jour appliquée et vérifiée." "\"$(date -Iseconds)\""
 echo "== done $(date -Iseconds) ==" >> "$LOG"