From f544b736f8227cea41ec213bcca0816c53c97fbb Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 8 Oct 2026 13:42:01 -0400 Subject: [PATCH 1/6] ci: guard the sacct node fallback, note why preflight matches SIGILL text The faulted-node sacct pipeline ran under set -e with no || true, so a failing sacct ended the script on the node-recovery path. Match the other sacct call. Also say why preflight matches "Illegal instruction" output. --- .github/scripts/preflight.sh | 3 ++- .github/scripts/submit-slurm-job.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/preflight.sh b/.github/scripts/preflight.sh index 2b017075f6..e5dc652698 100755 --- a/.github/scripts/preflight.sh +++ b/.github/scripts/preflight.sh @@ -151,7 +151,8 @@ run_probe() { # reasons -- no input file, no restart data, a missing module -- and none of them # say anything about the node. Treating any non-zero status as a fault here would # exclude every healthy node in the cluster. 132 is 128+4, a child killed by -# SIGILL; bash reports signals that way, and mpirun/srun forward it. +# SIGILL; bash reports signals that way, and mpirun/srun forward it. Some +# launchers do not pass 132 on, so the "Illegal instruction" text is matched too. isa_probe() { isa_bin=$(newest_for_device pre_process) [ -n "$isa_bin" ] || return 0 diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index 1db3bb5306..5736550f58 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -322,7 +322,7 @@ while :; do # 34183404644 exactly this way. sacct knows the node whether or not the # .out exists, so identification no longer depends on the marker. if [ -z "$faulted_node" ]; then - faulted_node=$(sacct -j "$job_id" -X -n -o NodeList 2>/dev/null | head -n1 | tr -d ' ') + faulted_node=$(sacct -j "$job_id" -X -n -o NodeList 2>/dev/null | head -n1 | tr -d ' ' || true) case "$faulted_node" in ""|None*|*[,\[]*) faulted_node="" ;; esac fi if [ "$node_attempt" -lt "$MFC_MAX_NODE_RESUBMITS" ]; then From 4d633907b6964801edd27c6272c65f5476527c9f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 8 Oct 2026 13:42:02 -0400 Subject: [PATCH 2/6] cmake: limit the LINK_DEPENDS_USE_LINKER opt-out to amdflang offload builds Only the offload link goes through clang-linker-wrapper, so CPU builds with LLVMFlang keep linker-reported dependencies. --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5495f62088..de51f4894d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,7 +116,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/cce") endif() -if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang") +if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang" AND MFC_OpenMP) # Otherwise the GPU targets are never up to date and every build redoes the # whole-image device LTO link -- ~20 minutes on Frontier, paid on each mfc.sh # invocation even when nothing changed. CMake asks the linker for its @@ -133,7 +133,8 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang") # Directory scope, not a target property: LINK_DEPENDS_USE_LINKER exists only as # a variable, and set_target_properties would silently create a custom property # that does nothing. Setting it inside MFC_SETUP_TARGET would be function-scoped - # and equally silent. + # and equally silent. MFC_OpenMP is the amdflang offload build; CPU builds keep + # the linker's dependency tracking. set(CMAKE_LINK_DEPENDS_USE_LINKER OFF) endif() From e7fbaf217a8b336ba6c63e159e1f3d6c9427abf6 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 8 Oct 2026 13:42:02 -0400 Subject: [PATCH 3/6] toolchain: tidy grcbc_in required-state check and compiler probe - Build the grcbc_in required inflow fields as one list. - Default optional EOS coefficients only when unset (is None), not when falsy. - test_thermochem: split the Cray flags dict; key the cached compiler probe on $FC. --- toolchain/mfc/case_validator.py | 10 +++++----- toolchain/mfc/test_thermochem.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/toolchain/mfc/case_validator.py b/toolchain/mfc/case_validator.py index 36fce10a00..8d6c2c2c44 100644 --- a/toolchain/mfc/case_validator.py +++ b/toolchain/mfc/case_validator.py @@ -1073,7 +1073,7 @@ def _eos_coefficient_args(self, i, family): args = [] for suffix in family.coefficients_args: value = self.get(f"fluid_pp({i})%{family.prefix}_{suffix}") - args.append((value or 0.0) if suffix in optional else value) + args.append(0.0 if value is None and suffix in optional else value) return args def _check_initial_states_inside_eos(self, num_fluids): @@ -1960,10 +1960,10 @@ def check_grcbc(self): # dir_idx, which is (2,1,3) for a y inflow and (3,1,2) for z -- so requiring only # component 1 would leave the normal velocity of a y or z inflow unchecked. num_dims = 3 if (self.get("p", 0) or 0) > 0 else (2 if (self.get("n", 0) or 0) > 0 else 1) - missing = [n for n in (f"bc_{dir}%pres_in",) if self.get(n) is None] - missing += [f"bc_{dir}%vel_in({d})" for d in range(1, num_dims + 1) if self.get(f"bc_{dir}%vel_in({d})") is None] - missing += [f"bc_{dir}%alpha_rho_in({i})" for i in range(1, num_fluids + 1) if self.get(f"bc_{dir}%alpha_rho_in({i})") is None] - missing += [f"bc_{dir}%alpha_in({i})" for i in range(1, num_fluids + 1) if self.get(f"bc_{dir}%alpha_in({i})") is None] + required = [f"bc_{dir}%pres_in"] + required += [f"bc_{dir}%vel_in({d})" for d in range(1, num_dims + 1)] + required += [f"bc_{dir}%{name}({i})" for name in ("alpha_rho_in", "alpha_in") for i in range(1, num_fluids + 1)] + missing = [name for name in required if self.get(name) is None] self.prohibit(len(missing) > 0, f"Subsonic Inflow (grcbc_in) needs the full inflow state; missing {', '.join(missing)}") if grcbc_out: # Check if EITHER beg OR end is set to -8 diff --git a/toolchain/mfc/test_thermochem.py b/toolchain/mfc/test_thermochem.py index d5b0389629..22c4a1ddf7 100644 --- a/toolchain/mfc/test_thermochem.py +++ b/toolchain/mfc/test_thermochem.py @@ -67,14 +67,24 @@ FPE_TRAP: ["-ffpe-trap=invalid,zero,overflow"], }, # CCE enables OpenACC by default; turn it off unless asked for, as cmake/MFCTargets.cmake does. - "Cray": {"base": ["-eZ"], None: ["-hnoacc"], "acc": ["-hacc"], "mp": ["-hnoacc", "-fopenmp"], FPE_TRAP: ["-Ktrap=divz,inv,ovf"]}, + "Cray": { + "base": ["-eZ"], + None: ["-hnoacc"], + "acc": ["-hacc"], + "mp": ["-hnoacc", "-fopenmp"], + FPE_TRAP: ["-Ktrap=divz,inv,ovf"], + }, } -@functools.cache def fortran_compiler(): """The compiler MFC's build would use ($FC, else ftn or gfortran on PATH) and its family, or None.""" - names = [os.environ["FC"]] if os.environ.get("FC") else ["ftn", "gfortran"] + return _probe_compiler(os.environ.get("FC") or None) + + +@functools.cache # keyed on $FC, so a changed $FC is probed afresh +def _probe_compiler(fc): + names = [fc] if fc else ["ftn", "gfortran"] for name in names: path = shutil.which(name) if path is None: From e68dee7d73bd4640b3e545586230ad80ce342ceb Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 8 Oct 2026 13:42:02 -0400 Subject: [PATCH 4/6] examples: derive SUMMARY topology, drop unused LEVELS 3D_ibm_neighborhood_radius takes the rank topology from TOPOLOGY (default 16,2,2) and uses the thinnest rank in any direction. 2D_ibm_thin_plate_force drops the unused LEVELS dict. --- examples/2D_ibm_thin_plate_force/case.py | 5 +---- examples/3D_ibm_neighborhood_radius/case.py | 7 +++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/2D_ibm_thin_plate_force/case.py b/examples/2D_ibm_thin_plate_force/case.py index b984ba5deb..bf73849a2a 100644 --- a/examples/2D_ibm_thin_plate_force/case.py +++ b/examples/2D_ibm_thin_plate_force/case.py @@ -33,14 +33,11 @@ t_end = t0 + t_p + 4.0 x0, x1, y0, y1 = -2.0, 5.0, -2.5, 2.5 -# L4 added after the first three failed to converge: refining 2 -> 4 -> 8 cells across the thickness moved +# NCELL=16 added after 2, 4 and 8 failed to converge: refining 2 -> 4 -> 8 cells across the thickness moved # the peak lift 6.41 -> 4.68 -> 4.46 against a reference of 7.00, i.e. away from it and then stalling. # Two under-resolved answers landing near each other is not convergence. If a few cells across a thin # body is simply too few for the immersed boundary, 16 should move back toward the reference; if the # finite thickness is genuinely the difference, it should stay near 4.5. -LEVELS = {"L1": 0.0125, "L2": 0.00625, "L3": 0.003125, "L4": 0.0015625} - - NCELL = int(os.environ.get("NCELL", 4)) dx = THICK / NCELL m, n = int((x1 - x0) / dx) - 1, int((y1 - y0) / dx) - 1 diff --git a/examples/3D_ibm_neighborhood_radius/case.py b/examples/3D_ibm_neighborhood_radius/case.py index 1669458c65..aa54adbd9f 100644 --- a/examples/3D_ibm_neighborhood_radius/case.py +++ b/examples/3D_ibm_neighborhood_radius/case.py @@ -126,7 +126,10 @@ if os.environ.get("SUMMARY"): bound = 0.5 * math.sqrt(c**2 + SPAN**2 + THICK**2) print(f"plate half-extent (s_get_ib_bound, geometry 9): {bound:.4f}") - print(f"rank extents at 64 ranks (16 x 2 x 2): x {(x1 - x0) / 16:.3f}, " f"y {(y1 - y0) / 2:.3f}, z {(z1 - z0) / 2:.3f}") - print(f"hops needed across the thinnest rank: ceil(1.1 * {bound:.4f} / {(x1 - x0) / 16:.3f}) = " f"{max(1, math.ceil(1.1 * bound / ((x1 - x0) / 16)))}") + # Ranks per direction, as MFC's topology search picks them for 400 x 50 x 50 at 64 ranks; set TOPOLOGY otherwise. + topo = [int(t) for t in os.environ.get("TOPOLOGY", "16,2,2").split(",")] + ext = [(hi - lo) / t for (lo, hi), t in zip(((x0, x1), (y0, y1), (z0, z1)), topo)] + print(f"rank extents at {topo[0] * topo[1] * topo[2]} ranks ({' x '.join(map(str, topo))}): x {ext[0]:.3f}, y {ext[1]:.3f}, z {ext[2]:.3f}") + print(f"hops needed across the thinnest rank: ceil(1.1 * {bound:.4f} / {min(ext):.3f}) = {max(1, math.ceil(1.1 * bound / min(ext)))}") else: print(json.dumps(case, indent=4)) From 479acdb6f63e9be44146b9a5fd1960959f8a7e0f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 8 Oct 2026 13:42:02 -0400 Subject: [PATCH 5/6] docs: column-align the showcase list; inflow ramp start is approximate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit f(t0) is frac0 only to within ~0.25%·(1 - frac0). The index.html change is whitespace only. --- docs/documentation/case.md | 2 +- docs/index.html | 80 +++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/documentation/case.md b/docs/documentation/case.md index 4efcc06e03..e0855e74a1 100644 --- a/docs/documentation/case.md +++ b/docs/documentation/case.md @@ -1405,7 +1405,7 @@ The entries labeled "Characteristic." are characteristic boundary conditions bas This boundary condition can be used for subsonic inflow (`bc_[x,y,z]%[beg,end]` = -7) and subsonic outflow (`bc_[x,y,z]%[beg,end]` = -8) characteristic boundary conditions. These are based on \cite Pirozzoli13. This enables to provide inflow and outflow conditions outside the computational domain. -`bc_[x,y,z]%%vel_in_ramp` starts the inflow smoothly instead of holding it constant, which is what a jet or a tunnel accelerating from rest requires: the start-up is the event of interest, not a transient to be discarded. The inflow velocity is scaled by \f$f(t) = f_0 + (1 - f_0)\left[1 + \tanh\left(6 (t - t_0)/\tau - 3\right)\right]/2\f$, with \f$\tau\f$ = `vel_in_ramp`, \f$t_0\f$ = `vel_in_t0` and \f$f_0\f$ = `vel_in_frac0`, so it leaves \f$f_0\f$ of the final velocity at \f$t_0\f$ and is within half a percent of it at \f$t_0 + \tau\f$. A boundary with `vel_in_ramp = 0` is held constant, as before. The ramp acts on a GRCBC inflow (`grcbc_in`) or on the ghost-cell velocity of a Dirichlet (-17) boundary or boundary patch normal to that direction, such as a nozzle set into a wall. +`bc_[x,y,z]%%vel_in_ramp` starts the inflow smoothly instead of holding it constant, which is what a jet or a tunnel accelerating from rest requires: the start-up is the event of interest, not a transient to be discarded. The inflow velocity is scaled by \f$f(t) = f_0 + (1 - f_0)\left[1 + \tanh\left(6 (t - t_0)/\tau - 3\right)\right]/2\f$, with \f$\tau\f$ = `vel_in_ramp`, \f$t_0\f$ = `vel_in_t0` and \f$f_0\f$ = `vel_in_frac0`, so it leaves \f$\approx f_0\f$ of the final velocity at \f$t_0\f$ and is within half a percent of it at \f$t_0 + \tau\f$. A boundary with `vel_in_ramp = 0` is held constant, as before. The ramp acts on a GRCBC inflow (`grcbc_in`) or on the ghost-cell velocity of a Dirichlet (-17) boundary or boundary patch normal to that direction, such as a nozzle set into a wall. ### Patch types {#patch-types} diff --git a/docs/index.html b/docs/index.html index 6987653079..a3e8117935 100644 --- a/docs/index.html +++ b/docs/index.html @@ -48,52 +48,52 @@