Preserve restart checkpoints and IB diagnostic history

docs/documentation/case.md

@@ -866,7 +866,9 @@ row    = t_step / ib_force_stride - t_step_start / ib_force_stride - 1
 offset = (row * num_ibs + ib_id - 1) * 353
 ```
 
-Rows count from the first step the run records, not from `t_step`, so row 0 is the first row of the file whether the run starts at step 0 or resumes from a restart. The first recorded step is the first multiple of `ib_force_stride` after `t_step_start`; `t_step_start` itself is skipped, because at that point the force is still the one from before the run began.
+Rows count from the first step the run records, not from `t_step`, so row 0 is the first row of the file whether the run starts at step 0 or resumes from a restart. The first recorded step is the first multiple of `ib_force_stride` after `t_step_start`; `t_step_start` itself is skipped, because at that point the force is still the one from before the run began. The last row is the run's final step if that step is a multiple of `ib_force_stride`, and otherwise the last multiple before it.
+
+A resumed run writes `D/ib_forces_<t_step_start>.dat` (`D/ib_forces_n<n_start>.dat` with `cfl_dt`) instead of overwriting `D/ib_forces.dat`. When each run resumes from the final step of the run before it, the files concatenate in order with no row repeated and none of the `ib_force_stride` sampling missing. This holds for any stride with a fixed `dt`. With `cfl_dt` the step counter restarts at 0 in each run, so only `ib_force_stride = 1` gives an unbroken history; a larger stride can leave up to `ib_force_stride - 1` unrecorded steps at the end of each run. A run killed mid-way has no final row and may hold rows past the restart it is resumed from.
 
 Rows are written in global body-id order, so the file is byte-identical however the domain is decomposed, and no merge step is needed after a parallel run.
 

src/common/m_helper_basic.fpp

@@ -15,7 +15,7 @@ module m_helper_basic
 
     private
     public :: f_approx_equal, f_approx_in_array, f_is_default, f_all_default, f_is_integer, s_configure_coordinate_bounds, &
-        & s_update_cell_bounds
+        & s_update_cell_bounds, f_save_index
 
 contains
 
@@ -109,6 +109,17 @@ contains
 
     end function f_is_integer
 
+    !> Index of the save written at t_step: t_step itself under CFL-based stepping (the caller passes the save count), else
+    !! t_step/t_step_save.
+    pure integer function f_save_index(t_step, cfl_mode, t_step_save) result(idx)
+
+        integer, intent(in) :: t_step, t_step_save
+        logical, intent(in) :: cfl_mode
+
+        idx = merge(t_step, t_step/t_step_save, cfl_mode)
+
+    end function f_save_index
+
     !> Compute ghost-cell buffer size and set interior/buffered coordinate index bounds.
     subroutine s_configure_coordinate_bounds(recon_type, weno_polyn, muscl_polyn, igr_order, buff_size, idwint, idwbuff, viscous, &
         & bubbles_lagrange, m, n, p, num_dims, igr, ib, fd_number)

src/post_process/m_data_input.f90

@@ -17,6 +17,7 @@ module m_data_input
     use m_boundary_common
     use m_boundary_io
     use m_helper
+    use m_helper_basic, only: f_save_index
 
     implicit none
 
@@ -153,7 +154,7 @@ impure subroutine s_read_ib_data_files(file_loc_base, t_step)
                 p_MOK = int(p_glb + 1, MPI_OFFSET_KIND)
                 MOK = int(1._wp, MPI_OFFSET_KIND)
                 WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
-                save_index = t_step/t_step_save  ! get the number of saves done to this point
+                save_index = f_save_index(t_step, cfl_dt, t_step_save)
 
                 data_size = (m + 1)*(n + 1)*(p + 1)
                 var_MOK = int(sys_size + 1, MPI_OFFSET_KIND)

src/simulation/m_data_output.fpp

@@ -1103,7 +1103,7 @@ contains
             call s_check_mpi_file_open(ierr, file_loc)
 
             var_MOK = int(sys_size + 1, MPI_OFFSET_KIND)
-            disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1 + int(time_step/t_step_save))
+            disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1 + f_save_index(time_step, cfl_dt, t_step_save))
             if (time_step == 0) disp = 0
 
             call MPI_FILE_SET_VIEW(ifile, disp, MPI_INTEGER, MPI_IO_IB_DATA%view, 'native', mpi_info_int, ierr)
@@ -1370,7 +1370,14 @@ contains
         @:PROHIBIT(len_trim(probe) /= IB_REC_BODY, &
                    & "IB force record width disagrees with IB_REC_BODY;  IB_REC_FMT and IB_REC_BODY must be changed together")
 
-        file_loc = trim(case_dir) // '/D/ib_forces.dat'
+        ! A resumed run writes its own file rather than overwriting the history of the runs before it
+        if (cfl_dt .and. n_start > 0) then
+            write (file_loc, '(A,I0,A)') trim(case_dir) // '/D/ib_forces_n', n_start, '.dat'
+        else if (t_step_start > 0) then
+            write (file_loc, '(A,I0,A)') trim(case_dir) // '/D/ib_forces_', t_step_start, '.dat'
+        else
+            file_loc = trim(case_dir) // '/D/ib_forces.dat'
+        end if
 #ifdef MFC_MPI
         ! MPI_MODE_CREATE does not truncate, so a shorter run following a longer one in the same
         ! directory would keep the old tail past its last record. Delete first, as the ib_state

src/simulation/p_main.fpp

@@ -8,18 +8,20 @@ program p_main
     use m_global_parameters
     use m_start_up
     use m_time_steppers
+    use m_data_output, only: s_write_ib_force_history
     use m_nvtx
 
     implicit none
 
-    integer                             :: t_step  !< Iterator for the time-stepping loop
+    integer                             :: t_step     !< Iterator for the time-stepping loop
     real(wp)                            :: time_avg, time_final
     real(wp)                            :: io_time_avg, io_time_final
     real(wp), allocatable, dimension(:) :: proc_time
     real(wp), allocatable, dimension(:) :: io_proc_time
     logical                             :: file_exists
     real(wp)                            :: start, finish
     integer                             :: nt
+    integer                             :: last_save  !< Index of the last CFL-mode save
 
     call system_clock(COUNT=cpu_start, COUNT_RATE=cpu_rate)
 
@@ -47,6 +49,7 @@ program p_main
         t_step = 0
         t_step_start = 0
         mytime = t_save*n_start
+        last_save = n_start
     else
         t_step = t_step_start
         if (t_step == 0) then
@@ -78,8 +81,10 @@ program p_main
         call s_perform_time_step(t_step, time_avg)
 
         if (cfl_dt) then
-            if (abs(mod(mytime, t_save)) < dt .or. mytime >= t_stop) then
+            ! Save once per index; a mod(mytime, t_save) < dt test can refire on a restart and overwrite n_start
+            if (int(mytime/t_save) > last_save .or. mytime >= t_stop) then
                 call s_save_data(t_step, start, finish, io_time_avg, nt)
+                last_save = int(mytime/t_save)
             end if
         else
             if (mod(t_step - t_step_start, t_step_save) == 0 .or. t_step == t_step_stop) then
@@ -90,6 +95,9 @@ program p_main
         call system_clock(cpu_end)
     end do
 
+    ! Step N's IB force record is written at the start of step N, which the loop never reaches for its last step
+    if (ib_state_wrt) call s_write_ib_force_history(t_step)
+
     call nvtxEndRange  ! Simulation
 
     deallocate (proc_time, io_proc_time)