Skip to content

Commit ac08ffe

Browse files
sbryngelsonsbryngelson
authored
ci: fail fast on bad nodes and cluster outages (#1797)
1 parent 30e7004 commit ac08ffe

34 files changed

Lines changed: 2366 additions & 40 deletions

‎.github/scripts/ci-outage.sh‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/bash
2+
# Per-cluster circuit breaker for environment-wide outages.
3+
#
4+
# Some failures are not the node's fault and not the code's fault: pypi.org
5+
# unreachable from a login node, a module tree mid-upgrade, a full project
6+
# filesystem. Requeuing elsewhere cannot help, and every job that starts pays
7+
# the same discovery cost -- on 2026-08-28, 17 Frontier jobs each spent ~33
8+
# minutes learning that PyPI was down.
9+
#
10+
# The first job to notice records a marker on the shared filesystem (every
11+
# self-hosted runner for a cluster shares $HOME); later jobs check it and exit
12+
# immediately instead of submitting a SLURM job that is going to fail.
13+
#
14+
# The breaker is deliberately self-healing. A marker expires after
15+
# MFC_CI_OUTAGE_TTL_SECONDS, and a marker that cannot be parsed is ignored, so
16+
# neither a stale file nor a truncated write can wedge CI. Only jobs that
17+
# actually observe the outage re-mark it, so once the outage clears the breaker
18+
# closes on its own.
19+
#
20+
# Usage:
21+
# ci-outage.sh mark <cluster> <reason> record an outage
22+
# ci-outage.sh check <cluster> exit 0 = clear, 1 = outage active
23+
# ci-outage.sh clear <cluster> reset the breaker
24+
#
25+
# Env:
26+
# MFC_CI_STATE_DIR where markers live (default ~/.mfc-ci-state)
27+
# MFC_CI_OUTAGE_TTL_SECONDS marker lifetime in seconds (default 1200)
28+
29+
set -uo pipefail
30+
31+
STATE_DIR="${MFC_CI_STATE_DIR:-$HOME/.mfc-ci-state}"
32+
TTL="${MFC_CI_OUTAGE_TTL_SECONDS:-1200}"
33+
34+
# A non-numeric TTL would make the age comparison below emit "integer expression
35+
# expected" and exit with a code the caller reads as neither clear nor tripped.
36+
# Fall back to the default rather than letting a typo gate CI.
37+
case "$TTL" in
38+
''|*[!0-9]*)
39+
echo "Ignoring non-numeric MFC_CI_OUTAGE_TTL_SECONDS='$TTL'; using 1200." >&2
40+
TTL=1200
41+
;;
42+
esac
43+
44+
EXIT_CLEAR=0
45+
EXIT_TRIPPED=1
46+
EXIT_USAGE=2
47+
48+
usage() {
49+
echo "Usage: $0 {mark <cluster> <reason>|check <cluster>|clear <cluster>}" >&2
50+
}
51+
52+
# Keep the marker name filesystem-safe regardless of what the caller passes.
53+
marker_for() {
54+
local cluster
55+
cluster=$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '_')
56+
printf '%s/outage-%s' "$STATE_DIR" "$cluster"
57+
}
58+
59+
cmd="${1:-}"
60+
cluster="${2:-}"
61+
62+
if [ -z "$cmd" ] || [ -z "$cluster" ]; then
63+
usage
64+
exit $EXIT_USAGE
65+
fi
66+
67+
marker=$(marker_for "$cluster")
68+
69+
case "$cmd" in
70+
mark)
71+
reason="${3:-unspecified}"
72+
mkdir -p "$STATE_DIR" || exit $EXIT_USAGE
73+
# Write to a temporary file and rename so a concurrent `check` never
74+
# observes a half-written marker.
75+
tmp="${marker}.$$.tmp"
76+
{
77+
date +%s
78+
printf '%s\n' "$reason"
79+
} > "$tmp" && mv -f "$tmp" "$marker"
80+
echo "Recorded $cluster outage: $reason"
81+
echo " marker: $marker (expires after ${TTL}s)"
82+
;;
83+
84+
check)
85+
[ -f "$marker" ] || exit $EXIT_CLEAR
86+
87+
stamp=$(head -n1 "$marker" 2>/dev/null)
88+
reason=$(tail -n +2 "$marker" 2>/dev/null)
89+
90+
# A marker we cannot parse is treated as absent: an unreadable breaker
91+
# must never be an un-clearable one.
92+
case "$stamp" in
93+
''|*[!0-9]*)
94+
echo "Ignoring unparseable outage marker $marker"
95+
exit $EXIT_CLEAR
96+
;;
97+
esac
98+
99+
age=$(( $(date +%s) - stamp ))
100+
if [ "$age" -ge "$TTL" ] || [ "$age" -lt 0 ]; then
101+
exit $EXIT_CLEAR
102+
fi
103+
104+
echo "::warning::Skipping: known $cluster outage recorded ${age}s ago: ${reason:-unspecified}"
105+
echo "Clear it early by deleting $marker"
106+
exit $EXIT_TRIPPED
107+
;;
108+
109+
clear)
110+
rm -f "$marker"
111+
echo "Cleared any $cluster outage marker ($marker)"
112+
;;
113+
114+
*)
115+
usage
116+
exit $EXIT_USAGE
117+
;;
118+
esac

‎.github/scripts/classify-build-failure.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
# Decide whether a failed build was a cluster-wide dependency outage.
3+
#
4+
# MFC bootstraps its Python toolchain into build/venv on the first ./mfc.sh call
5+
# of a job, pulling from pypi.org. On Phoenix clean_build has just moved build/
6+
# aside, so that happens every time; on Frontier it happens in the login-node
7+
# "Fetch Dependencies" step. When the index is unreachable the build fails for a
8+
# reason no other node improves on, so it is worth recording once and skipping
9+
# the rest of the matrix rather than having each job spend ~33 minutes
10+
# rediscovering it (17 Frontier jobs did exactly that on 2026-08-28).
11+
#
12+
# Usage: classify-build-failure.sh <logfile> <cluster>
13+
#
14+
# Exit codes:
15+
# 78 cluster-wide dependency outage; it has been recorded
16+
# 0 ordinary build failure, caller should keep its own exit code
17+
18+
set -uo pipefail
19+
20+
log="${1:-}"
21+
cluster="${2:-}"
22+
23+
if [ -z "$log" ] || [ -z "$cluster" ]; then
24+
echo "Usage: $0 <logfile> <cluster>" >&2
25+
exit 0
26+
fi
27+
28+
# No log means nothing to classify. Never claim an outage on absent evidence.
29+
[ -f "$log" ] || exit 0
30+
31+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
32+
33+
# The URL may or may not be wrapped (uv quotes it in backticks, plain pip does
34+
# not), so do not require a character between the colon and the scheme.
35+
if grep -qE "Failed to fetch:[^h]*https?://pypi|uv install failed|\(venv\) Installation failed" "$log"; then
36+
bash "$SCRIPT_DIR/ci-outage.sh" mark "$cluster" \
37+
"PyPI/uv dependency install failed during build"
38+
exit 78
39+
fi
40+
41+
exit 0

‎.github/scripts/monitor_slurm_job.sh‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,11 @@ if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then
9999
echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2
100100
exit 1
101101
fi
102+
# How long to wait between status polls and between output-stabilization
103+
# checks. Overridable so tests can exercise this script without sleeping
104+
# through it; CI leaves it at the default.
105+
: "${MFC_MONITOR_POLL_SECONDS:=5}"
106+
102107
queue_start=$(date +%s)
103108

104109
abort_queue_starvation() {
@@ -142,7 +147,7 @@ while [ ! -f "$output_file" ]; do
142147
;;
143148
PENDING|CONFIGURING)
144149
unknown_count=0
145-
sleep 5
150+
sleep "$MFC_MONITOR_POLL_SECONDS"
146151
;;
147152
RUNNING|COMPLETING)
148153
unknown_count=0
@@ -155,7 +160,7 @@ while [ ! -f "$output_file" ]; do
155160
if [ $((unknown_count % 12)) -eq 1 ]; then
156161
echo "Warning: Could not query job $job_id state (SLURM may be temporarily unavailable)..."
157162
fi
158-
sleep 5
163+
sleep "$MFC_MONITOR_POLL_SECONDS"
159164
;;
160165
*)
161166
# Terminal state — job finished without creating output
@@ -164,7 +169,7 @@ while [ ! -f "$output_file" ]; do
164169
exit 1
165170
fi
166171
# Unrecognized state, keep waiting
167-
sleep 5
172+
sleep "$MFC_MONITOR_POLL_SECONDS"
168173
;;
169174
esac
170175
done
@@ -205,7 +210,7 @@ while true; do
205210
last_heartbeat=$current_time
206211
fi
207212

208-
sleep 5
213+
sleep "$MFC_MONITOR_POLL_SECONDS"
209214
done
210215

211216
# Give tail a moment to flush the final lines, then stop streaming.
@@ -229,7 +234,7 @@ if [ -f "$output_file" ]; then
229234
if [ $same_count -ge 2 ]; then
230235
break
231236
fi
232-
sleep 5
237+
sleep "$MFC_MONITOR_POLL_SECONDS"
233238
done
234239
fi
235240

@@ -262,6 +267,23 @@ if [ -z "$exit_code" ]; then
262267
exit 1
263268
fi
264269

270+
# Infrastructure verdicts from the in-allocation preflight come back as the
271+
# job's own exit code. Relay them verbatim: flattening them to 1 would leave the
272+
# submit wrapper unable to tell "this node is unusable" (exclude it and try
273+
# again) from "the tests failed" (report it).
274+
case "$exit_code" in
275+
77:*)
276+
echo "Job $job_id failed preflight: the node is unusable — signaling caller to exclude it and resubmit."
277+
monitor_success=1
278+
exit 77
279+
;;
280+
78:*)
281+
echo "Job $job_id skipped: a cluster-wide outage is already recorded."
282+
monitor_success=1
283+
exit 78
284+
;;
285+
esac
286+
265287
# Check if job succeeded
266288
if [ "$exit_code" != "0:0" ]; then
267289
echo "ERROR: Job $job_id failed with exit code $exit_code"

‎.github/scripts/node-exclude.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
# Bookkeeping for the sbatch --exclude list used when a node fails preflight.
3+
#
4+
# The in-allocation preflight prints "MFC_FAULT_NODE=<name>" into the job's
5+
# output file when it finds the node unusable (dead GPU, SIGILL from a binary
6+
# built for another microarchitecture). The submit wrapper reads that back, adds
7+
# the node to --exclude and resubmits, so the retry lands elsewhere.
8+
#
9+
# Bad nodes are concentrated rather than scattered -- over 2026-08-18..31 one
10+
# Phoenix node accounted for 25 of 29 ECC failures -- which is why excluding the
11+
# offender is worth doing and why it was previously a hand-edited constant.
12+
#
13+
# Usage:
14+
# node-exclude.sh node-from <output-file> print the faulted node, if any
15+
# node-exclude.sh merge <csv> <node> print <csv> with <node> added once
16+
17+
set -uo pipefail
18+
19+
usage() {
20+
echo "Usage: $0 {node-from <output-file>|merge <csv> <node>}" >&2
21+
}
22+
23+
case "${1:-}" in
24+
node-from)
25+
file="${2:-}"
26+
[ -f "$file" ] || exit 0
27+
# Last marker wins: one output path is reused across resubmits, so an
28+
# earlier attempt's marker can still be sitting above the current one.
29+
sed -n 's/.*MFC_FAULT_NODE=\([A-Za-z0-9._-]\{1,\}\).*/\1/p' "$file" | tail -n1
30+
;;
31+
32+
merge)
33+
csv="${2-}"
34+
node="${3-}"
35+
if [ -z "$node" ]; then
36+
printf '%s\n' "$csv"
37+
exit 0
38+
fi
39+
if [ -z "$csv" ]; then
40+
printf '%s\n' "$node"
41+
exit 0
42+
fi
43+
# Wrapping both sides in commas compares whole fields, so a shorter name
44+
# that happens to be a prefix of the new one is not mistaken for a match.
45+
case ",$csv," in
46+
*",$node,"*) printf '%s\n' "$csv" ;;
47+
*) printf '%s,%s\n' "$csv" "$node" ;;
48+
esac
49+
;;
50+
51+
*)
52+
usage
53+
exit 2
54+
;;
55+
esac

‎.github/scripts/prebuild-case-optimization.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,15 @@ if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
9090
fi
9191
fi
9292

93+
# Deliberately no node probe here. This pre-build is submitted as a *cpu*
94+
# allocation (see test.yml: it is --dry-run, so it only builds), while the
95+
# binaries it produces are GPU builds. syscheck built with --gpu therefore
96+
# asserts omp_get_num_devices() > 0 and exits non-zero on a node that has no
97+
# GPU by design -- which a probe would report as a bad node. It did: three
98+
# healthy Phoenix nodes were condemned and two excluded before the wrapper gave
99+
# up. The GPU allocation that actually runs these cases is probed instead, in
100+
# run_case_optimization.sh.
101+
93102
idx=0
94103
for case in "${benchmarks[@]}"; do
95104
idx=$((idx + 1))

0 commit comments

Comments
 (0)