Skip to content

Commit a0e3ddc

Browse files
danieljvickersdanieljvickers
authored
1538 ib state file per process io (#1540)
1 parent c9dd5c5 commit a0e3ddc

3 files changed

Lines changed: 189 additions & 78 deletions

File tree

‎src/post_process/m_data_output.fpp‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1356,6 +1356,8 @@ contains
13561356
integer, dimension(num_procs) :: meshtypes
13571357
integer :: i, ios, file_unit
13581358
integer :: ierr, nBodies
1359+
integer :: r, nlocal, gbl_id
1360+
character(len=10) :: t_step_string
13591361
real(wp), dimension(:), allocatable :: px, py, pz
13601362
real(wp), dimension(:), allocatable :: force_x, force_y, force_z
13611363
real(wp), dimension(:), allocatable :: torque_x, torque_y, torque_z
@@ -1365,15 +1367,6 @@ contains
13651367
real(wp), dimension(:), allocatable :: ib_diameter
13661368
13671369
if (proc_rank == 0) then
1368-
! Build path to per-timestep IB state file
1369-
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
1370-
file_loc = trim(case_dir) // trim(file_loc)
1371-
1372-
inquire (FILE=trim(file_loc), EXIST=file_exist)
1373-
if (.not. file_exist) then
1374-
call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!')
1375-
end if
1376-
13771370
nBodies = num_ibs
13781371
13791372
if (nBodies > 0) then
@@ -1386,16 +1379,51 @@ contains
13861379
allocate (angle_x(nBodies), angle_y(nBodies), angle_z(nBodies))
13871380
allocate (ib_diameter(nBodies))
13881381
1389-
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
1390-
if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc))
1382+
if (file_per_process) then
1383+
call s_int_to_str(t_step, t_step_string)
1384+
ib_data = 0._wp
1385+
do r = 0, num_procs - 1
1386+
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', r, '.dat'
1387+
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)
13911388
1392-
do i = 1, nBodies
1393-
read (file_unit, iostat=ios) ib_buf
1394-
if (ios /= 0) call s_mpi_abort('Error reading IB state file')
1395-
ib_data(i,:) = ib_buf(:)
1396-
end do
1389+
inquire (FILE=trim(file_loc), EXIST=file_exist)
1390+
if (.not. file_exist) call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!')
1391+
1392+
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
1393+
if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc))
1394+
1395+
read (file_unit, iostat=ios) nlocal
1396+
if (ios /= 0) call s_mpi_abort('Error reading IB state file header: ' // trim(file_loc))
1397+
1398+
do i = 1, nlocal
1399+
read (file_unit, iostat=ios) gbl_id
1400+
if (ios /= 0) call s_mpi_abort('Error reading IB patch ID: ' // trim(file_loc))
1401+
read (file_unit, iostat=ios) ib_buf
1402+
if (ios /= 0) call s_mpi_abort('Error reading IB state data: ' // trim(file_loc))
1403+
ib_data(gbl_id,:) = ib_buf(:)
1404+
end do
13971405
1398-
close (file_unit)
1406+
close (file_unit)
1407+
end do
1408+
else
1409+
! Build path to per-timestep IB state file
1410+
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
1411+
file_loc = trim(case_dir) // trim(file_loc)
1412+
1413+
inquire (FILE=trim(file_loc), EXIST=file_exist)
1414+
if (.not. file_exist) call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!')
1415+
1416+
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
1417+
if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc))
1418+
1419+
do i = 1, nBodies
1420+
read (file_unit, iostat=ios) ib_buf
1421+
if (ios /= 0) call s_mpi_abort('Error reading IB state file')
1422+
ib_data(i,:) = ib_buf(:)
1423+
end do
1424+
1425+
close (file_unit)
1426+
end if
13991427
14001428
do i = 1, nBodies
14011429
force_x(i) = ib_data(i, 2); force_y(i) = ib_data(i, 3); force_z(i) = ib_data(i, 4)

‎src/simulation/m_data_output.fpp‎

Lines changed: 79 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,10 @@ contains
753753
end if
754754

755755
call MPI_FILE_CLOSE(ifile, ierr)
756+
757+
if (ib) then
758+
call s_write_parallel_ib_data(t_step)
759+
end if
756760
else
757761
if (ib) then
758762
call s_initialize_mpi_data(q_cons_vf, ib_markers)
@@ -908,52 +912,95 @@ contains
908912
integer(kind=MPI_OFFSET_KIND) :: WP_MOK
909913
integer :: ifile, ierr
910914
integer, dimension(MPI_STATUS_SIZE) :: status
911-
logical :: file_exist
915+
logical :: file_exist, dir_check
912916
integer :: i, ib_idx
913917
integer, parameter :: NFIELDS_PER_IB = 20
914918
real(wp) :: ib_buf(NFIELDS_PER_IB)
919+
integer :: file_unit
920+
character(len=10) :: t_step_string
915921

916922
! Partition IBs across ranks round-robin style
917923
integer :: ib_start, ib_end, nibs_per_rank, remainder
918924

919925
WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND)
920926

921-
if (proc_rank == 0) then
922-
call s_create_directory(trim(case_dir) // '/restart_data')
923-
end if
924-
call s_mpi_barrier()
927+
if (file_per_process) then
928+
call s_int_to_str(t_step, t_step_string)
925929

926-
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
927-
file_loc = trim(case_dir) // trim(file_loc)
930+
if (proc_rank == 0) then
931+
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string)
932+
call s_create_directory(trim(file_loc))
933+
end if
934+
call s_mpi_barrier()
935+
call DelayFileAccess(proc_rank)
928936

929-
inquire (FILE=trim(file_loc), EXIST=file_exist)
930-
if (file_exist .and. proc_rank == 0) then
931-
call MPI_FILE_DELETE(file_loc, mpi_info_int, ierr)
932-
end if
933-
call s_mpi_barrier()
937+
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', proc_rank, '.dat'
938+
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)
934939

935-
call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)
940+
inquire (FILE=trim(file_loc), EXIST=file_exist)
941+
if (file_exist) then
942+
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='replace')
943+
else
944+
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='new')
945+
end if
936946

937-
do i = 1, num_local_ibs
938-
ib_idx = local_ib_patch_ids(i)
939-
ib_buf(1) = mytime
940-
ib_buf(2:4) = patch_ib(ib_idx)%force(1:3)
941-
ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3)
942-
ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3)
943-
ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3)
944-
ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3)
945-
ib_buf(17) = patch_ib(ib_idx)%x_centroid
946-
ib_buf(18) = patch_ib(ib_idx)%y_centroid
947-
ib_buf(19) = patch_ib(ib_idx)%z_centroid
948-
ib_buf(20) = patch_ib(ib_idx)%radius
949-
950-
! Global IB index (i) determines position in file
951-
disp = int(patch_ib(ib_idx)%gbl_patch_id - 1, MPI_OFFSET_KIND)*int(NFIELDS_PER_IB, MPI_OFFSET_KIND)*WP_MOK
952-
953-
call MPI_FILE_WRITE_AT(ifile, disp, ib_buf, NFIELDS_PER_IB, mpi_p, status, ierr)
954-
end do
947+
write (file_unit) num_local_ibs
948+
do i = 1, num_local_ibs
949+
ib_idx = local_ib_patch_ids(i)
950+
ib_buf(1) = mytime
951+
ib_buf(2:4) = patch_ib(ib_idx)%force(1:3)
952+
ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3)
953+
ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3)
954+
ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3)
955+
ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3)
956+
ib_buf(17) = patch_ib(ib_idx)%x_centroid
957+
ib_buf(18) = patch_ib(ib_idx)%y_centroid
958+
ib_buf(19) = patch_ib(ib_idx)%z_centroid
959+
ib_buf(20) = patch_ib(ib_idx)%radius
960+
961+
write (file_unit) patch_ib(ib_idx)%gbl_patch_id
962+
write (file_unit) ib_buf
963+
end do
955964

956-
call MPI_FILE_CLOSE(ifile, ierr)
965+
close (file_unit)
966+
else
967+
if (proc_rank == 0) then
968+
call s_create_directory(trim(case_dir) // '/restart_data')
969+
end if
970+
call s_mpi_barrier()
971+
972+
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
973+
file_loc = trim(case_dir) // trim(file_loc)
974+
975+
inquire (FILE=trim(file_loc), EXIST=file_exist)
976+
if (file_exist .and. proc_rank == 0) then
977+
call MPI_FILE_DELETE(file_loc, mpi_info_int, ierr)
978+
end if
979+
call s_mpi_barrier()
980+
981+
call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)
982+
983+
do i = 1, num_local_ibs
984+
ib_idx = local_ib_patch_ids(i)
985+
ib_buf(1) = mytime
986+
ib_buf(2:4) = patch_ib(ib_idx)%force(1:3)
987+
ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3)
988+
ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3)
989+
ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3)
990+
ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3)
991+
ib_buf(17) = patch_ib(ib_idx)%x_centroid
992+
ib_buf(18) = patch_ib(ib_idx)%y_centroid
993+
ib_buf(19) = patch_ib(ib_idx)%z_centroid
994+
ib_buf(20) = patch_ib(ib_idx)%radius
995+
996+
! Global IB index determines position in file
997+
disp = int(patch_ib(ib_idx)%gbl_patch_id - 1, MPI_OFFSET_KIND)*int(NFIELDS_PER_IB, MPI_OFFSET_KIND)*WP_MOK
998+
999+
call MPI_FILE_WRITE_AT(ifile, disp, ib_buf, NFIELDS_PER_IB, mpi_p, status, ierr)
1000+
end do
1001+
1002+
call MPI_FILE_CLOSE(ifile, ierr)
1003+
end if
9571004
#endif
9581005

9591006
end subroutine s_write_parallel_ib_state

‎src/simulation/m_start_up.fpp‎

Lines changed: 65 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1114,47 +1114,83 @@ contains
11141114
integer, intent(in) :: t_step
11151115
character(len=path_len + 2*name_len) :: file_loc
11161116
integer :: i, ios, file_unit, ierr
1117+
integer :: r, nlocal, gbl_id
11171118
integer, parameter :: NFIELDS_PER_IB = 20
11181119
real(wp) :: ib_buf(NFIELDS_PER_IB)
11191120
logical :: file_exist
1121+
character(len=10) :: t_step_string
11201122

1121-
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
1122-
file_loc = trim(case_dir) // trim(file_loc)
1123+
if (file_per_process) then
1124+
call s_int_to_str(t_step, t_step_string)
11231125

1124-
if (proc_rank == 0) then
1125-
inquire (FILE=trim(file_loc), EXIST=file_exist)
1126-
if (.not. file_exist) then
1127-
call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc))
1128-
end if
1126+
do r = 0, num_procs - 1
1127+
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', r, '.dat'
1128+
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)
11291129

1130-
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
1131-
if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc))
1130+
inquire (FILE=trim(file_loc), EXIST=file_exist)
1131+
if (.not. file_exist) call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc))
1132+
1133+
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
1134+
if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc))
1135+
1136+
read (file_unit, iostat=ios) nlocal
1137+
if (ios /= 0) call s_mpi_abort('Error reading IB state file header: ' // trim(file_loc))
1138+
1139+
do i = 1, nlocal
1140+
read (file_unit, iostat=ios) gbl_id
1141+
if (ios /= 0) call s_mpi_abort('Error reading IB patch ID: ' // trim(file_loc))
1142+
read (file_unit, iostat=ios) ib_buf
1143+
if (ios /= 0) call s_mpi_abort('Error reading IB state data: ' // trim(file_loc))
1144+
1145+
patch_ib(gbl_id)%vel = ib_buf(8:10)
1146+
patch_ib(gbl_id)%angular_vel = ib_buf(11:13)
1147+
patch_ib(gbl_id)%angles = ib_buf(14:16)
1148+
patch_ib(gbl_id)%x_centroid = ib_buf(17)
1149+
patch_ib(gbl_id)%y_centroid = ib_buf(18)
1150+
patch_ib(gbl_id)%z_centroid = ib_buf(19)
1151+
end do
11321152

1133-
do i = 1, num_ibs
1134-
read (file_unit, iostat=ios) ib_buf
1135-
if (ios /= 0) call s_mpi_abort('Error reading IB state restart file')
1136-
1137-
patch_ib(i)%vel = ib_buf(8:10)
1138-
patch_ib(i)%angular_vel = ib_buf(11:13)
1139-
patch_ib(i)%angles = ib_buf(14:16)
1140-
patch_ib(i)%x_centroid = ib_buf(17)
1141-
patch_ib(i)%y_centroid = ib_buf(18)
1142-
patch_ib(i)%z_centroid = ib_buf(19)
1153+
close (file_unit)
11431154
end do
1155+
else
1156+
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
1157+
file_loc = trim(case_dir) // trim(file_loc)
11441158

1145-
close (file_unit)
1146-
end if
1159+
if (proc_rank == 0) then
1160+
inquire (FILE=trim(file_loc), EXIST=file_exist)
1161+
if (.not. file_exist) then
1162+
call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc))
1163+
end if
1164+
1165+
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
1166+
if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc))
1167+
1168+
do i = 1, num_ibs
1169+
read (file_unit, iostat=ios) ib_buf
1170+
if (ios /= 0) call s_mpi_abort('Error reading IB state restart file')
1171+
1172+
patch_ib(i)%vel = ib_buf(8:10)
1173+
patch_ib(i)%angular_vel = ib_buf(11:13)
1174+
patch_ib(i)%angles = ib_buf(14:16)
1175+
patch_ib(i)%x_centroid = ib_buf(17)
1176+
patch_ib(i)%y_centroid = ib_buf(18)
1177+
patch_ib(i)%z_centroid = ib_buf(19)
1178+
end do
1179+
1180+
close (file_unit)
1181+
end if
11471182

11481183
#ifdef MFC_MPI
1149-
do i = 1, num_ibs
1150-
call MPI_BCAST(patch_ib(i)%vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
1151-
call MPI_BCAST(patch_ib(i)%angular_vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
1152-
call MPI_BCAST(patch_ib(i)%angles, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
1153-
call MPI_BCAST(patch_ib(i)%x_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
1154-
call MPI_BCAST(patch_ib(i)%y_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
1155-
call MPI_BCAST(patch_ib(i)%z_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
1156-
end do
1184+
do i = 1, num_ibs
1185+
call MPI_BCAST(patch_ib(i)%vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
1186+
call MPI_BCAST(patch_ib(i)%angular_vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
1187+
call MPI_BCAST(patch_ib(i)%angles, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
1188+
call MPI_BCAST(patch_ib(i)%x_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
1189+
call MPI_BCAST(patch_ib(i)%y_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
1190+
call MPI_BCAST(patch_ib(i)%z_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
1191+
end do
11571192
#endif
1193+
end if
11581194

11591195
end subroutine s_read_ib_restart_data
11601196

0 commit comments

Comments
 (0)