diff --git a/src/meshfem3D/create_central_cube_buffers.f90 b/src/meshfem3D/create_central_cube_buffers.f90 index 29f83bf12..fbe6ba065 100644 --- a/src/meshfem3D/create_central_cube_buffers.f90 +++ b/src/meshfem3D/create_central_cube_buffers.f90 @@ -61,7 +61,7 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & ! local to global mapping integer, dimension(NSPEC_INNER_CORE), intent(in) :: idoubling_inner_core - real(kind=CUSTOM_REAL), dimension(NGLOB_INNER_CORE), intent(in) :: xstore_inner_core,ystore_inner_core,zstore_inner_core + double precision, dimension(NGLOB_INNER_CORE), intent(in) :: xstore_inner_core,ystore_inner_core,zstore_inner_core ! boundary parameters locator integer, intent(in) :: nspec2D_xmin_inner_core,nspec2D_xmax_inner_core,nspec2D_ymin_inner_core,nspec2D_ymax_inner_core @@ -307,9 +307,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do i = 1,NGLLX ipoin = ipoin + 1 iglob = ibool_inner_core(i,j,k,ispec) - buffer_slices(ipoin,1) = dble(xstore_inner_core(iglob)) - buffer_slices(ipoin,2) = dble(ystore_inner_core(iglob)) - buffer_slices(ipoin,3) = dble(zstore_inner_core(iglob)) + buffer_slices(ipoin,1) = xstore_inner_core(iglob) + buffer_slices(ipoin,2) = ystore_inner_core(iglob) + buffer_slices(ipoin,3) = zstore_inner_core(iglob) enddo enddo enddo @@ -337,9 +337,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do i = 1,NGLLX ipoin = ipoin + 1 iglob = ibool_inner_core(i,j,k,ispec) - buffer_slices(ipoin,1) = dble(xstore_inner_core(iglob)) - buffer_slices(ipoin,2) = dble(ystore_inner_core(iglob)) - buffer_slices(ipoin,3) = dble(zstore_inner_core(iglob)) + buffer_slices(ipoin,1) = xstore_inner_core(iglob) + buffer_slices(ipoin,2) = ystore_inner_core(iglob) + buffer_slices(ipoin,3) = zstore_inner_core(iglob) enddo enddo endif @@ -384,9 +384,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do k = 1,NGLLZ do j = 1,NGLLY iglob = ibool_inner_core(i,j,k,ispec) - x_current = dble(xstore_inner_core(iglob)) - y_current = dble(ystore_inner_core(iglob)) - z_current = dble(zstore_inner_core(iglob)) + x_current = xstore_inner_core(iglob) + y_current = ystore_inner_core(iglob) + z_current = zstore_inner_core(iglob) ! look for matching point if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec) @@ -409,9 +409,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do k = 1,NGLLZ do j = 1,NGLLY iglob = ibool_inner_core(i,j,k,ispec) - x_current = dble(xstore_inner_core(iglob)) - y_current = dble(ystore_inner_core(iglob)) - z_current = dble(zstore_inner_core(iglob)) + x_current = xstore_inner_core(iglob) + y_current = ystore_inner_core(iglob) + z_current = zstore_inner_core(iglob) ! look for matching point if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec) @@ -434,9 +434,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do k = 1,NGLLZ do i = 1,NGLLX iglob = ibool_inner_core(i,j,k,ispec) - x_current = dble(xstore_inner_core(iglob)) - y_current = dble(ystore_inner_core(iglob)) - z_current = dble(zstore_inner_core(iglob)) + x_current = xstore_inner_core(iglob) + y_current = ystore_inner_core(iglob) + z_current = zstore_inner_core(iglob) ! look for matching point if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec) @@ -459,9 +459,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do k = 1,NGLLZ do i = 1,NGLLX iglob = ibool_inner_core(i,j,k,ispec) - x_current = dble(xstore_inner_core(iglob)) - y_current = dble(ystore_inner_core(iglob)) - z_current = dble(zstore_inner_core(iglob)) + x_current = xstore_inner_core(iglob) + y_current = ystore_inner_core(iglob) + z_current = zstore_inner_core(iglob) ! look for matching point if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec) @@ -479,9 +479,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do j = 1,NGLLY do i = 1,NGLLX iglob = ibool_inner_core(i,j,k,ispec) - x_current = dble(xstore_inner_core(iglob)) - y_current = dble(ystore_inner_core(iglob)) - z_current = dble(zstore_inner_core(iglob)) + x_current = xstore_inner_core(iglob) + y_current = ystore_inner_core(iglob) + z_current = zstore_inner_core(iglob) ! look for matching point if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec) @@ -524,9 +524,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, & do j = 1,NGLLY do i = 1,NGLLX iglob = ibool_inner_core(i,j,k,ispec) - x_current = dble(xstore_inner_core(iglob)) - y_current = dble(ystore_inner_core(iglob)) - z_current = dble(zstore_inner_core(iglob)) + x_current = xstore_inner_core(iglob) + y_current = ystore_inner_core(iglob) + z_current = zstore_inner_core(iglob) ! look for matching point if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then ibool_central_cube(imsg,ipoin) = 0 diff --git a/src/meshfem3D/create_regions_mesh.F90 b/src/meshfem3D/create_regions_mesh.F90 index d537ca921..3e67f847f 100644 --- a/src/meshfem3D/create_regions_mesh.F90 +++ b/src/meshfem3D/create_regions_mesh.F90 @@ -1378,11 +1378,11 @@ subroutine crm_fill_global_meshes() allocate(xstore_glob(nglob),ystore_glob(nglob),zstore_glob(nglob),stat=ier) if (ier /= 0) call exit_mpi(myrank,'Error allocating temporary global mesh arrays') - xstore_glob(:) = 0._CUSTOM_REAL - ystore_glob(:) = 0._CUSTOM_REAL - zstore_glob(:) = 0._CUSTOM_REAL + xstore_glob(:) = 0.d0 + ystore_glob(:) = 0.d0 + zstore_glob(:) = 0.d0 - ! fill CUSTOM_REAL arrays + ! fill arrays do ispec = 1,nspec do k = 1,NGLLZ do j = 1,NGLLY @@ -1394,10 +1394,10 @@ subroutine crm_fill_global_meshes() stop 'Error fill global meshes has invalid iglob index' endif - ! distinguish between single and double precision for reals - xstore_glob(iglob) = real(xstore(i,j,k,ispec), kind=CUSTOM_REAL) - ystore_glob(iglob) = real(ystore(i,j,k,ispec), kind=CUSTOM_REAL) - zstore_glob(iglob) = real(zstore(i,j,k,ispec), kind=CUSTOM_REAL) + ! store global node (last-only fill) + xstore_glob(iglob) = xstore(i,j,k,ispec) + ystore_glob(iglob) = ystore(i,j,k,ispec) + zstore_glob(iglob) = zstore(i,j,k,ispec) enddo enddo enddo diff --git a/src/meshfem3D/get_MPI_interfaces.f90 b/src/meshfem3D/get_MPI_interfaces.f90 index 1a3720250..8b52fca08 100644 --- a/src/meshfem3D/get_MPI_interfaces.f90 +++ b/src/meshfem3D/get_MPI_interfaces.f90 @@ -31,9 +31,9 @@ subroutine get_MPI_interfaces(myrank,NGLOB,NSPEC, & max_nibool,MAX_NEIGHBORS, & ibool,is_on_a_slice_edge, & IREGION,add_central_cube,idoubling,INCLUDE_CENTRAL_CUBE, & - xstore,ystore,zstore,NPROCTOT) + xstore_glob,ystore_glob,zstore_glob,NPROCTOT) - use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE + use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE,IMAIN implicit none integer,intent(in) :: myrank,NGLOB,NSPEC @@ -56,7 +56,7 @@ subroutine get_MPI_interfaces(myrank,NGLOB,NSPEC, & logical,intent(in) :: INCLUDE_CENTRAL_CUBE - real(kind=CUSTOM_REAL),dimension(NGLOB),intent(in) :: xstore,ystore,zstore + double precision,dimension(NGLOB),intent(in) :: xstore_glob,ystore_glob,zstore_glob integer,intent(in) :: NPROCTOT @@ -590,7 +590,7 @@ subroutine get_MPI_interfaces(myrank,NGLOB,NSPEC, & ! sort buffer obtained to be conforming with neighbor in other chunk npoin = nibool_neighbors(iinterface) call sort_MPI_interface(myrank,npoin,ibool_neighbors(1:npoin,iinterface), & - NGLOB,xstore,ystore,zstore) + NGLOB,xstore_glob,ystore_glob,zstore_glob) enddo @@ -608,7 +608,7 @@ end subroutine get_MPI_interfaces ! subroutine sort_MPI_interface(myrank,npoin,ibool_n, & - NGLOB,xstore,ystore,zstore) + NGLOB,xstore_glob,ystore_glob,zstore_glob) use constants, only: CUSTOM_REAL @@ -618,7 +618,7 @@ subroutine sort_MPI_interface(myrank,npoin,ibool_n, & integer,dimension(npoin),intent(inout) :: ibool_n integer,intent(in) :: NGLOB - real(kind=CUSTOM_REAL), dimension(NGLOB) :: xstore,ystore,zstore + double precision,dimension(NGLOB),intent(in) :: xstore_glob,ystore_glob,zstore_glob ! local parameters ! arrays for sorting routine @@ -639,17 +639,22 @@ subroutine sort_MPI_interface(myrank,npoin,ibool_n, & ifseg(npoin),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error sort MPI interface: allocating temporary sorting arrays') - ! sets up working arrays + ! sets up working arrays for interface points do i = 1,npoin ipoin = ibool_n(i) ibool_selected(i) = ipoin - xstore_selected(i) = dble(xstore(ipoin)) - ystore_selected(i) = dble(ystore(ipoin)) - zstore_selected(i) = dble(zstore(ipoin)) + xstore_selected(i) = xstore_glob(ipoin) + ystore_selected(i) = ystore_glob(ipoin) + zstore_selected(i) = zstore_glob(ipoin) enddo + ! note: the order produced here must match with the neighbor rank, + ! that is both sides must sort the coordinates into the same order. + ! + ! this is using locations from double-precision values for better accuracy. + ! sort buffer obtained to be conforming with neighbor in other chunk ! sort on x, y and z, the other arrays will be swapped as well call sort_array_coordinates(npoin,xstore_selected,ystore_selected,zstore_selected, & @@ -662,9 +667,8 @@ subroutine sort_MPI_interface(myrank,npoin,ibool_n, & ibool_n(1:npoin) = ibool_selected(1:npoin) ! frees array memory - deallocate(ibool_selected,xstore_selected,ystore_selected,zstore_selected, & - ninseg,iglob,locval,ifseg) - + deallocate(ibool_selected,xstore_selected,ystore_selected,zstore_selected) + deallocate(ninseg,iglob,locval,ifseg) end subroutine sort_MPI_interface diff --git a/src/meshfem3D/meshfem3D_par.f90 b/src/meshfem3D/meshfem3D_par.f90 index 5d7f18d08..1acff916c 100644 --- a/src/meshfem3D/meshfem3D_par.f90 +++ b/src/meshfem3D/meshfem3D_par.f90 @@ -218,7 +218,7 @@ module meshfem_par double precision, dimension(:,:,:,:), allocatable :: xstore,ystore,zstore ! temporary arrays with mesh only on global points - real(kind=CUSTOM_REAL), dimension(:),allocatable :: xstore_glob,ystore_glob,zstore_glob + double precision, dimension(:),allocatable :: xstore_glob,ystore_glob,zstore_glob ! parameters needed to store the radii of the grid points ! in the spherically symmetric Earth diff --git a/src/meshfem3D/setup_MPI_interfaces.f90 b/src/meshfem3D/setup_MPI_interfaces.f90 index 4bbd6566a..6d7b318b8 100644 --- a/src/meshfem3D/setup_MPI_interfaces.f90 +++ b/src/meshfem3D/setup_MPI_interfaces.f90 @@ -28,8 +28,7 @@ subroutine setup_MPI_interfaces(iregion_code) - use meshfem_par, only: & - INCLUDE_CENTRAL_CUBE,myrank,NUMFACES_SHARED + use meshfem_par, only: myrank,INCLUDE_CENTRAL_CUBE,NUMFACES_SHARED use MPI_interfaces_par @@ -52,7 +51,6 @@ subroutine setup_MPI_interfaces(iregion_code) integer, dimension(:,:),allocatable :: ibool_neighbors integer :: ier - ! allocates temporary arrays for setup routines ! estimates a maximum size of needed arrays MAX_NEIGHBORS = 8 + NCORNERSCHUNKS if (INCLUDE_CENTRAL_CUBE) MAX_NEIGHBORS = MAX_NEIGHBORS + NUMMSGS_FACES @@ -270,7 +268,7 @@ subroutine setup_MPI_interfaces_cm(MAX_NEIGHBORS,my_neighbors,nibool_neighbors, do i = 1,num_interfaces_crust_mantle write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_crust_mantle_proc',myrank, & '_',my_neighbors_crust_mantle(i) - call write_VTK_data_points(NGLOB_crust_mantle, & + call write_VTK_data_points(NGLOB_CRUST_MANTLE, & xstore_glob,ystore_glob,zstore_glob, & ibool_interfaces_crust_mantle(1:nibool_interfaces_crust_mantle(i),i), & nibool_interfaces_crust_mantle(i),filename) @@ -284,6 +282,13 @@ subroutine setup_MPI_interfaces_cm(MAX_NEIGHBORS,my_neighbors,nibool_neighbors, my_neighbors_crust_mantle,nibool_interfaces_crust_mantle, & ibool_interfaces_crust_mantle) + ! checks cross-rank interface correspondence + call test_MPI_interfaces_matching(IREGION_CRUST_MANTLE, & + num_interfaces_crust_mantle,max_nibool_interfaces_cm, & + my_neighbors_crust_mantle,nibool_interfaces_crust_mantle, & + ibool_interfaces_crust_mantle, & + NGLOB_CRUST_MANTLE,xstore_glob,ystore_glob,zstore_glob) + ! checks with assembly of test fields call test_MPI_cm() @@ -424,6 +429,13 @@ subroutine setup_MPI_interfaces_oc(MAX_NEIGHBORS,my_neighbors,nibool_neighbors, my_neighbors_outer_core,nibool_interfaces_outer_core, & ibool_interfaces_outer_core) + ! checks cross-rank interface correspondence + call test_MPI_interfaces_matching(IREGION_OUTER_CORE, & + num_interfaces_outer_core,max_nibool_interfaces_oc, & + my_neighbors_outer_core,nibool_interfaces_outer_core, & + ibool_interfaces_outer_core, & + NGLOB_OUTER_CORE,xstore_glob,ystore_glob,zstore_glob) + ! checks with assembly of test fields call test_MPI_oc() @@ -627,6 +639,13 @@ subroutine setup_MPI_interfaces_ic(MAX_NEIGHBORS,my_neighbors,nibool_neighbors, my_neighbors_inner_core,nibool_interfaces_inner_core, & ibool_interfaces_inner_core) + ! checks cross-rank interface correspondence + call test_MPI_interfaces_matching(IREGION_INNER_CORE, & + num_interfaces_inner_core,max_nibool_interfaces_ic, & + my_neighbors_inner_core,nibool_interfaces_inner_core, & + ibool_interfaces_inner_core, & + NGLOB_INNER_CORE,xstore_glob,ystore_glob,zstore_glob) + ! checks with assembly of test fields call test_MPI_ic() @@ -774,6 +793,13 @@ subroutine setup_MPI_interfaces_trinf(MAX_NEIGHBORS,my_neighbors,nibool_neighbor my_neighbors_trinfinite,nibool_interfaces_trinfinite, & ibool_interfaces_trinfinite) + ! checks cross-rank interface correspondence + call test_MPI_interfaces_matching(IREGION_TRINFINITE, & + num_interfaces_trinfinite,max_nibool_interfaces_trinfinite, & + my_neighbors_trinfinite,nibool_interfaces_trinfinite, & + ibool_interfaces_trinfinite, & + NGLOB_TRINFINITE,xstore_glob,ystore_glob,zstore_glob) + ! checks with assembly of test fields call test_MPI_trinf() @@ -914,6 +940,13 @@ subroutine setup_MPI_interfaces_inf(MAX_NEIGHBORS,my_neighbors,nibool_neighbors, my_neighbors_infinite,nibool_interfaces_infinite, & ibool_interfaces_infinite) + ! checks cross-rank interface correspondence + call test_MPI_interfaces_matching(IREGION_INFINITE, & + num_interfaces_infinite,max_nibool_interfaces_infinite, & + my_neighbors_infinite,nibool_interfaces_infinite, & + ibool_interfaces_infinite, & + NGLOB_INFINITE,xstore_glob,ystore_glob,zstore_glob) + ! checks with assembly of test fields call test_MPI_inf() diff --git a/src/meshfem3D/setup_inner_outer.f90 b/src/meshfem3D/setup_inner_outer.f90 index a8032017c..bf54c28fb 100644 --- a/src/meshfem3D/setup_inner_outer.f90 +++ b/src/meshfem3D/setup_inner_outer.f90 @@ -114,8 +114,8 @@ subroutine setup_inner_outer(iregion_code) if (DEBUG) then write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_crust_mantle_proc',myrank call write_VTK_data_elem_l(NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE, & - xstore_glob,ystore_glob,zstore_glob, & - ibool,is_on_a_slice_edge,filename) + xstore_glob,ystore_glob,zstore_glob, & + ibool,is_on_a_slice_edge,filename) endif !debug @@ -168,8 +168,8 @@ subroutine setup_inner_outer(iregion_code) if (DEBUG) then write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_outer_core_proc',myrank call write_VTK_data_elem_l(NSPEC_OUTER_CORE,NGLOB_OUTER_CORE, & - xstore_glob,ystore_glob,zstore_glob, & - ibool,is_on_a_slice_edge,filename) + xstore_glob,ystore_glob,zstore_glob, & + ibool,is_on_a_slice_edge,filename) endif case (IREGION_INNER_CORE) @@ -265,8 +265,8 @@ subroutine setup_inner_outer(iregion_code) if (DEBUG) then write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_inner_core_proc',myrank call write_VTK_data_elem_l(NSPEC_INNER_CORE,NGLOB_INNER_CORE, & - xstore_glob,ystore_glob,zstore_glob, & - ibool,is_on_a_slice_edge,filename) + xstore_glob,ystore_glob,zstore_glob, & + ibool,is_on_a_slice_edge,filename) endif case (IREGION_TRINFINITE) @@ -311,8 +311,8 @@ subroutine setup_inner_outer(iregion_code) if (DEBUG) then write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_trinfinite_proc',myrank call write_VTK_data_elem_l(NSPEC_TRINFINITE,NGLOB_TRINFINITE, & - xstore_glob,ystore_glob,zstore_glob, & - ibool,is_on_a_slice_edge,filename) + xstore_glob,ystore_glob,zstore_glob, & + ibool,is_on_a_slice_edge,filename) endif case (IREGION_INFINITE) @@ -357,8 +357,8 @@ subroutine setup_inner_outer(iregion_code) if (DEBUG) then write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_infinite_proc',myrank call write_VTK_data_elem_l(NSPEC_INFINITE,NGLOB_INFINITE, & - xstore_glob,ystore_glob,zstore_glob, & - ibool,is_on_a_slice_edge,filename) + xstore_glob,ystore_glob,zstore_glob, & + ibool,is_on_a_slice_edge,filename) endif end select diff --git a/src/meshfem3D/test_MPI_interfaces.f90 b/src/meshfem3D/test_MPI_interfaces.f90 index d9592510c..1c596fcfe 100644 --- a/src/meshfem3D/test_MPI_interfaces.f90 +++ b/src/meshfem3D/test_MPI_interfaces.f90 @@ -31,7 +31,8 @@ subroutine test_MPI_neighbors(iregion_code, & my_neighbors,nibool_interfaces, & ibool_interfaces) - use constants + use constants, only: IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE, & + IREGION_TRINFINITE,IREGION_INFINITE,IMAIN,itag use meshfem_par, only: NPROCTOT,myrank use MPI_crust_mantle_par, only: NGLOB_CRUST_MANTLE @@ -814,3 +815,222 @@ subroutine test_MPI_inf() call synchronize_all() end subroutine test_MPI_inf + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine test_MPI_interfaces_matching(iregion_code, & + num_interfaces,max_nibool_interfaces, & + my_neighbors,nibool_interfaces, & + ibool_interfaces, & + NGLOB,xstore_glob,ystore_glob,zstore_glob) + + use constants, only: myrank,CUSTOM_REAL,IMAIN,itag,SMALLVALTOL + use meshfem_par, only: NPROCTOT,R_PLANET + + implicit none + + integer,intent(in) :: iregion_code + integer,intent(in) :: num_interfaces,max_nibool_interfaces + integer,dimension(num_interfaces),intent(in) :: my_neighbors,nibool_interfaces + integer,dimension(max_nibool_interfaces,num_interfaces),intent(in) :: ibool_interfaces + + integer,intent(in) :: NGLOB + double precision,dimension(NGLOB),intent(in) :: xstore_glob,ystore_glob,zstore_glob + + ! local parameters + integer :: iinterface,ineighbor,npoin,npoin_other,ipoin,iglob,ier + integer, dimension(:), allocatable :: send_count,recv_count + integer, dimension(:), allocatable :: req_send,req_recv + double precision, dimension(:,:), allocatable :: send_coord,recv_coord + double precision :: dx,dy,dz,dist,max_dist,max_dist_all + + ! checks if anything to do + if (NPROCTOT <= 1) return + if (num_interfaces <= 0) return + + ! user output + if (myrank == 0) then + write(IMAIN,*) ' MPI interface matching:' + call flush_IMAIN() + endif + + ! initializes + allocate(send_count(num_interfaces), & + recv_count(num_interfaces), & + req_send(num_interfaces), & + req_recv(num_interfaces), stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error test MPI matching: allocating count buffers') + send_count(:) = 0; recv_count(:) = 0 + req_send(:) = 0; req_recv(:) = 0 + + ! send/receive point coordinates + allocate(send_coord(3*max_nibool_interfaces,num_interfaces), & + recv_coord(3*max_nibool_interfaces,num_interfaces),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error test MPI matching: allocating coordinate buffers') + send_coord(:,:) = 0.d0; recv_coord(:,:) = 0.d0 + + ! fills count/coord arrays + ! loops over MPI interfaces + do iinterface = 1,num_interfaces + ineighbor = my_neighbors(iinterface) + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + ! packs the sorted coordinates of the interface points + do ipoin = 1,npoin + iglob = ibool_interfaces(ipoin,iinterface) + send_coord(3*(ipoin-1) + 1,iinterface) = xstore_glob(iglob) + send_coord(3*(ipoin-1) + 2,iinterface) = ystore_glob(iglob) + send_coord(3*(ipoin-1) + 3,iinterface) = zstore_glob(iglob) + enddo + + ! interface point count + send_count(iinterface) = npoin + enddo + + ! exchanges counts + do iinterface = 1,num_interfaces + ineighbor = my_neighbors(iinterface) + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + ! exchanges the number of interface points first + call irecv_i(recv_count(iinterface),1,ineighbor,itag,req_recv(iinterface)) + call isend_i(send_count(iinterface),1,ineighbor,itag,req_send(iinterface)) + enddo + + ! wait for counts + do iinterface = 1,num_interfaces + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + call wait_req(req_recv(iinterface)) + call wait_req(req_send(iinterface)) + enddo + + ! checks interface point count + do iinterface = 1,num_interfaces + ineighbor = my_neighbors(iinterface) + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + ! neighbor count + npoin_other = recv_count(iinterface) + + ! check number of interface points + if (npoin_other /= npoin) then + print *,'Error test MPI matching: rank ',myrank,' neighbor ',ineighbor + print *,' number of interface points: ',npoin,' (mine) ',npoin_other,' (neighbor)' + call exit_mpi(myrank,'Error test MPI: number of interface points differs') + endif + enddo + + ! synchronizes all MPI processes + call synchronize_all() + + ! user output + if (myrank == 0) then + write(IMAIN,*) ' point counts match' + call flush_IMAIN() + endif + + ! exchanges the sorted coordinates with the neighbor + do iinterface = 1,num_interfaces + ineighbor = my_neighbors(iinterface) + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + ! exchange coordinates + call irecv_dp(recv_coord(1:3*npoin,iinterface),3*npoin,ineighbor,itag,req_recv(iinterface)) + call isend_dp(send_coord(1:3*npoin,iinterface),3*npoin,ineighbor,itag,req_send(iinterface)) + enddo + + ! wait for coords + do iinterface = 1,num_interfaces + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + call wait_req(req_recv(iinterface)) + call wait_req(req_send(iinterface)) + enddo + + ! node distance to neighboring slice + max_dist = 0.d0 + + ! checks interface point coordinates + do iinterface = 1,num_interfaces + ineighbor = my_neighbors(iinterface) + npoin = nibool_interfaces(iinterface) + + ! skip if no interface points + if (npoin <= 0) cycle + + ! compares the sorted coordinates point by point + do ipoin = 1,npoin + iglob = ibool_interfaces(ipoin,iinterface) + + dx = xstore_glob(iglob) - recv_coord(3*(ipoin-1) + 1,iinterface) + dy = ystore_glob(iglob) - recv_coord(3*(ipoin-1) + 2,iinterface) + dz = zstore_glob(iglob) - recv_coord(3*(ipoin-1) + 3,iinterface) + + ! distance + dist = sqrt(dx*dx + dy*dy + dz*dz) + + ! overall maximum distance + if (dist > max_dist) max_dist = dist + + ! check distance + if (dist > SMALLVALTOL) then + print *,'Error test MPI matching: rank ',myrank,' neighbor ',ineighbor,' region ',iregion_code + print *,' interface point ',ipoin,' match distance (to the neighbor point) ',dist, & + ' exceeds check tolerance ',SMALLVALTOL + print *,' location (mine) : x/y/z = ',xstore_glob(iglob),'/', & + ystore_glob(iglob),'/', & + zstore_glob(iglob) + print *,' (neighbor): x/y/z = ',recv_coord(3*(ipoin-1)+1,iinterface),'/', & + recv_coord(3*(ipoin-1)+2,iinterface),'/', & + recv_coord(3*(ipoin-1)+3,iinterface) + call exit_mpi(myrank,'Error test MPI: interface points do not match') + endif + enddo + enddo + + ! synchronizes all MPI processes + call synchronize_all() + + ! report the worst-case distance across ranks (tracking, not fatal) + call max_all_dp(max_dist,max_dist_all) + + ! user output + if (myrank == 0) then + write(IMAIN,*) ' maximum shared point distance = ',max_dist_all + write(IMAIN,*) ' = ',max_dist_all * R_PLANET,'(m)' + write(IMAIN,*) ' interface match is okay' + write(IMAIN,*) + call flush_IMAIN() + endif + + ! free temporary arrays + deallocate(send_count,recv_count) + deallocate(req_send,req_recv) + deallocate(send_coord,recv_coord) + + ! synchronizes all MPI processes + call synchronize_all() + + end subroutine test_MPI_interfaces_matching + diff --git a/src/shared/parallel.f90 b/src/shared/parallel.f90 index 0c242fd81..b9d093689 100644 --- a/src/shared/parallel.f90 +++ b/src/shared/parallel.f90 @@ -814,6 +814,23 @@ subroutine min_all_all_dp(sendbuf, recvbuf) end subroutine min_all_all_dp +! +!------------------------------------------------------------------------------------------------- +! + + subroutine min_all_dp(sendbuf, recvbuf) + + use my_mpi + + implicit none + + double precision :: sendbuf, recvbuf + integer :: ier + + call MPI_REDUCE(sendbuf,recvbuf,1,MPI_DOUBLE_PRECISION,MPI_MIN,0,my_local_mpi_comm_world,ier) + + end subroutine min_all_dp + ! !------------------------------------------------------------------------------------------------- ! @@ -919,13 +936,6 @@ subroutine max_all_all_cr(sendbuf, recvbuf) end subroutine max_all_all_cr -! -!------------------------------------------------------------------------------------------------- -! - -! subroutine min_all_dp(sendbuf, recvbuf) -! end subroutine min_all_dp - ! !------------------------------------------------------------------------------------------------- ! diff --git a/src/shared/write_VTK_file.f90 b/src/shared/write_VTK_file.f90 index 498dd95f0..b6deb2c0d 100644 --- a/src/shared/write_VTK_file.f90 +++ b/src/shared/write_VTK_file.f90 @@ -68,7 +68,7 @@ subroutine write_VTK_data_points(nglob, & integer,intent(in) :: nglob ! global coordinates - real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy + double precision, dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy ! GLL data values array integer,intent(in) :: num_points_globalindices @@ -129,7 +129,7 @@ end subroutine write_VTK_data_points ! integer,intent(in) :: nglob ! ! ! global coordinates -! real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy +! double precision, dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy ! ! ! GLL data values array ! real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: glob_values @@ -178,8 +178,8 @@ end subroutine write_VTK_data_points subroutine write_VTK_data_elem_l(nspec,nglob, & - xstore_dummy,ystore_dummy,zstore_dummy,ibool, & - elem_flag,prname_file) + xstore_dummy,ystore_dummy,zstore_dummy,ibool, & + elem_flag,prname_file) ! routine for saving VTK file holding logical flag on each spectral element @@ -191,7 +191,7 @@ subroutine write_VTK_data_elem_l(nspec,nglob, & ! global coordinates integer, dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: ibool - real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy + double precision, dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy ! element flag array logical, dimension(nspec),intent(in) :: elem_flag @@ -270,7 +270,7 @@ subroutine write_VTK_data_elem_i(nspec,nglob, & ! global coordinates integer, dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: ibool - real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy + double precision, dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy ! element flag array integer, dimension(nspec),intent(in) :: elem_flag @@ -862,7 +862,7 @@ subroutine write_VTK_data_elem_cr(nspec,nglob, & ! global coordinates integer, dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: ibool - real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy + double precision, dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy ! element data values array real(kind=CUSTOM_REAL), dimension(nspec),intent(in) :: elem_data @@ -930,7 +930,7 @@ subroutine write_VTU_data_elem_cr_binary(nspec,nglob, & ! global coordinates integer, dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: ibool - real(kind=CUSTOM_REAL), dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy + double precision, dimension(nglob),intent(in) :: xstore_dummy,ystore_dummy,zstore_dummy ! element data values array real(kind=CUSTOM_REAL), dimension(nspec),intent(in) :: elem_data diff --git a/src/tomography/postprocess_sensitivity_kernels/smooth_sem.F90 b/src/tomography/postprocess_sensitivity_kernels/smooth_sem.F90 index f3016146d..b383eab27 100644 --- a/src/tomography/postprocess_sensitivity_kernels/smooth_sem.F90 +++ b/src/tomography/postprocess_sensitivity_kernels/smooth_sem.F90 @@ -198,6 +198,7 @@ program smooth_sem_globe ! debugging integer, dimension(:),allocatable :: ispec_flag !real(kind=CUSTOM_REAL), dimension(:,:,:,:),allocatable :: tmp_bk + double precision, dimension(:), allocatable :: tmp_xstore_glob, tmp_ystore_glob, tmp_zstore_glob double precision :: memory_size ! timing @@ -1163,10 +1164,15 @@ program smooth_sem_globe ! writes out vtk file for debugging write(filename,'(a,i4.4,a,i6.6)') trim(LOCAL_PATH)//'/search_elem',tmp_ispec_dbg,'_proc',iproc - call write_VTK_data_elem_i(NSPEC_AB,NGLOB_AB,xstore,ystore,zstore, & + + allocate(tmp_xstore_glob(NGLOB_AB),tmp_ystore_glob(NGLOB_AB),tmp_zstore_glob(NGLOB_AB)) + tmp_xstore_glob = dble(xstore); tmp_ystore_glob = dble(ystore); tmp_zstore_glob = dble(zstore) + + call write_VTK_data_elem_i(NSPEC_AB,NGLOB_AB,tmp_xstore_glob,tmp_ystore_glob,tmp_zstore_glob, & ibool,ispec_flag,filename) print *,'file written: ',trim(filename)//'.vtk' + deallocate(tmp_xstore_glob,tmp_ystore_glob,tmp_zstore_glob) deallocate(ispec_flag) endif endif