Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions src/meshfem3D/create_central_cube_buffers.f90
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
! local to global mapping
integer, dimension(NSPEC_INNER_CORE), intent(in) :: idoubling_inner_core

real(kind=CUSTOM_REAL), dimension(NGLOB_INNER_CORE), intent(in) :: xstore_inner_core,ystore_inner_core,zstore_inner_core
double precision, dimension(NGLOB_INNER_CORE), intent(in) :: xstore_inner_core,ystore_inner_core,zstore_inner_core

! boundary parameters locator
integer, intent(in) :: nspec2D_xmin_inner_core,nspec2D_xmax_inner_core,nspec2D_ymin_inner_core,nspec2D_ymax_inner_core
Expand Down Expand Up @@ -307,9 +307,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do i = 1,NGLLX
ipoin = ipoin + 1
iglob = ibool_inner_core(i,j,k,ispec)
buffer_slices(ipoin,1) = dble(xstore_inner_core(iglob))
buffer_slices(ipoin,2) = dble(ystore_inner_core(iglob))
buffer_slices(ipoin,3) = dble(zstore_inner_core(iglob))
buffer_slices(ipoin,1) = xstore_inner_core(iglob)
buffer_slices(ipoin,2) = ystore_inner_core(iglob)
buffer_slices(ipoin,3) = zstore_inner_core(iglob)
enddo
enddo
enddo
Expand Down Expand Up @@ -337,9 +337,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do i = 1,NGLLX
ipoin = ipoin + 1
iglob = ibool_inner_core(i,j,k,ispec)
buffer_slices(ipoin,1) = dble(xstore_inner_core(iglob))
buffer_slices(ipoin,2) = dble(ystore_inner_core(iglob))
buffer_slices(ipoin,3) = dble(zstore_inner_core(iglob))
buffer_slices(ipoin,1) = xstore_inner_core(iglob)
buffer_slices(ipoin,2) = ystore_inner_core(iglob)
buffer_slices(ipoin,3) = zstore_inner_core(iglob)
enddo
enddo
endif
Expand Down Expand Up @@ -384,9 +384,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do k = 1,NGLLZ
do j = 1,NGLLY
iglob = ibool_inner_core(i,j,k,ispec)
x_current = dble(xstore_inner_core(iglob))
y_current = dble(ystore_inner_core(iglob))
z_current = dble(zstore_inner_core(iglob))
x_current = xstore_inner_core(iglob)
y_current = ystore_inner_core(iglob)
z_current = zstore_inner_core(iglob)
! look for matching point
if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then
ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec)
Expand All @@ -409,9 +409,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do k = 1,NGLLZ
do j = 1,NGLLY
iglob = ibool_inner_core(i,j,k,ispec)
x_current = dble(xstore_inner_core(iglob))
y_current = dble(ystore_inner_core(iglob))
z_current = dble(zstore_inner_core(iglob))
x_current = xstore_inner_core(iglob)
y_current = ystore_inner_core(iglob)
z_current = zstore_inner_core(iglob)
! look for matching point
if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then
ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec)
Expand All @@ -434,9 +434,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do k = 1,NGLLZ
do i = 1,NGLLX
iglob = ibool_inner_core(i,j,k,ispec)
x_current = dble(xstore_inner_core(iglob))
y_current = dble(ystore_inner_core(iglob))
z_current = dble(zstore_inner_core(iglob))
x_current = xstore_inner_core(iglob)
y_current = ystore_inner_core(iglob)
z_current = zstore_inner_core(iglob)
! look for matching point
if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then
ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec)
Expand All @@ -459,9 +459,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do k = 1,NGLLZ
do i = 1,NGLLX
iglob = ibool_inner_core(i,j,k,ispec)
x_current = dble(xstore_inner_core(iglob))
y_current = dble(ystore_inner_core(iglob))
z_current = dble(zstore_inner_core(iglob))
x_current = xstore_inner_core(iglob)
y_current = ystore_inner_core(iglob)
z_current = zstore_inner_core(iglob)
! look for matching point
if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then
ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec)
Expand All @@ -479,9 +479,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do j = 1,NGLLY
do i = 1,NGLLX
iglob = ibool_inner_core(i,j,k,ispec)
x_current = dble(xstore_inner_core(iglob))
y_current = dble(ystore_inner_core(iglob))
z_current = dble(zstore_inner_core(iglob))
x_current = xstore_inner_core(iglob)
y_current = ystore_inner_core(iglob)
z_current = zstore_inner_core(iglob)
! look for matching point
if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then
ibool_central_cube(imsg,ipoin) = ibool_inner_core(i,j,k,ispec)
Expand Down Expand Up @@ -524,9 +524,9 @@ subroutine create_central_cube_buffers(iproc_xi,iproc_eta,ichunk, &
do j = 1,NGLLY
do i = 1,NGLLX
iglob = ibool_inner_core(i,j,k,ispec)
x_current = dble(xstore_inner_core(iglob))
y_current = dble(ystore_inner_core(iglob))
z_current = dble(zstore_inner_core(iglob))
x_current = xstore_inner_core(iglob)
y_current = ystore_inner_core(iglob)
z_current = zstore_inner_core(iglob)
! look for matching point
if (dsqrt((x_current-x_target)**2 + (y_current-y_target)**2 + (z_current-z_target)**2) < SMALLVALTOL) then
ibool_central_cube(imsg,ipoin) = 0
Expand Down
16 changes: 8 additions & 8 deletions src/meshfem3D/create_regions_mesh.F90
Original file line number Diff line number Diff line change
Expand Up @@ -1378,11 +1378,11 @@ subroutine crm_fill_global_meshes()
allocate(xstore_glob(nglob),ystore_glob(nglob),zstore_glob(nglob),stat=ier)
if (ier /= 0) call exit_mpi(myrank,'Error allocating temporary global mesh arrays')

xstore_glob(:) = 0._CUSTOM_REAL
ystore_glob(:) = 0._CUSTOM_REAL
zstore_glob(:) = 0._CUSTOM_REAL
xstore_glob(:) = 0.d0
ystore_glob(:) = 0.d0
zstore_glob(:) = 0.d0

! fill CUSTOM_REAL arrays
! fill arrays
do ispec = 1,nspec
do k = 1,NGLLZ
do j = 1,NGLLY
Expand All @@ -1394,10 +1394,10 @@ subroutine crm_fill_global_meshes()
stop 'Error fill global meshes has invalid iglob index'
endif

! distinguish between single and double precision for reals
xstore_glob(iglob) = real(xstore(i,j,k,ispec), kind=CUSTOM_REAL)
ystore_glob(iglob) = real(ystore(i,j,k,ispec), kind=CUSTOM_REAL)
zstore_glob(iglob) = real(zstore(i,j,k,ispec), kind=CUSTOM_REAL)
! store global node (last-only fill)
xstore_glob(iglob) = xstore(i,j,k,ispec)
ystore_glob(iglob) = ystore(i,j,k,ispec)
zstore_glob(iglob) = zstore(i,j,k,ispec)
enddo
enddo
enddo
Expand Down
30 changes: 17 additions & 13 deletions src/meshfem3D/get_MPI_interfaces.f90
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ subroutine get_MPI_interfaces(myrank,NGLOB,NSPEC, &
max_nibool,MAX_NEIGHBORS, &
ibool,is_on_a_slice_edge, &
IREGION,add_central_cube,idoubling,INCLUDE_CENTRAL_CUBE, &
xstore,ystore,zstore,NPROCTOT)
xstore_glob,ystore_glob,zstore_glob,NPROCTOT)

use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE
use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE,IMAIN
implicit none

integer,intent(in) :: myrank,NGLOB,NSPEC
Expand All @@ -56,7 +56,7 @@ subroutine get_MPI_interfaces(myrank,NGLOB,NSPEC, &

logical,intent(in) :: INCLUDE_CENTRAL_CUBE

real(kind=CUSTOM_REAL),dimension(NGLOB),intent(in) :: xstore,ystore,zstore
double precision,dimension(NGLOB),intent(in) :: xstore_glob,ystore_glob,zstore_glob

integer,intent(in) :: NPROCTOT

Expand Down Expand Up @@ -590,7 +590,7 @@ subroutine get_MPI_interfaces(myrank,NGLOB,NSPEC, &
! sort buffer obtained to be conforming with neighbor in other chunk
npoin = nibool_neighbors(iinterface)
call sort_MPI_interface(myrank,npoin,ibool_neighbors(1:npoin,iinterface), &
NGLOB,xstore,ystore,zstore)
NGLOB,xstore_glob,ystore_glob,zstore_glob)

enddo

Expand All @@ -608,7 +608,7 @@ end subroutine get_MPI_interfaces
!

subroutine sort_MPI_interface(myrank,npoin,ibool_n, &
NGLOB,xstore,ystore,zstore)
NGLOB,xstore_glob,ystore_glob,zstore_glob)

use constants, only: CUSTOM_REAL

Expand All @@ -618,7 +618,7 @@ subroutine sort_MPI_interface(myrank,npoin,ibool_n, &
integer,dimension(npoin),intent(inout) :: ibool_n

integer,intent(in) :: NGLOB
real(kind=CUSTOM_REAL), dimension(NGLOB) :: xstore,ystore,zstore
double precision,dimension(NGLOB),intent(in) :: xstore_glob,ystore_glob,zstore_glob

! local parameters
! arrays for sorting routine
Expand All @@ -639,17 +639,22 @@ subroutine sort_MPI_interface(myrank,npoin,ibool_n, &
ifseg(npoin),stat=ier)
if (ier /= 0 ) call exit_MPI(myrank,'Error sort MPI interface: allocating temporary sorting arrays')

! sets up working arrays
! sets up working arrays for interface points
do i = 1,npoin
ipoin = ibool_n(i)

ibool_selected(i) = ipoin

xstore_selected(i) = dble(xstore(ipoin))
ystore_selected(i) = dble(ystore(ipoin))
zstore_selected(i) = dble(zstore(ipoin))
xstore_selected(i) = xstore_glob(ipoin)
ystore_selected(i) = ystore_glob(ipoin)
zstore_selected(i) = zstore_glob(ipoin)
enddo

! note: the order produced here must match with the neighbor rank,
! that is both sides must sort the coordinates into the same order.
!
! this is using locations from double-precision values for better accuracy.

! sort buffer obtained to be conforming with neighbor in other chunk
! sort on x, y and z, the other arrays will be swapped as well
call sort_array_coordinates(npoin,xstore_selected,ystore_selected,zstore_selected, &
Expand All @@ -662,9 +667,8 @@ subroutine sort_MPI_interface(myrank,npoin,ibool_n, &
ibool_n(1:npoin) = ibool_selected(1:npoin)

! frees array memory
deallocate(ibool_selected,xstore_selected,ystore_selected,zstore_selected, &
ninseg,iglob,locval,ifseg)

deallocate(ibool_selected,xstore_selected,ystore_selected,zstore_selected)
deallocate(ninseg,iglob,locval,ifseg)

end subroutine sort_MPI_interface

Expand Down
2 changes: 1 addition & 1 deletion src/meshfem3D/meshfem3D_par.f90
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ module meshfem_par
double precision, dimension(:,:,:,:), allocatable :: xstore,ystore,zstore

! temporary arrays with mesh only on global points
real(kind=CUSTOM_REAL), dimension(:),allocatable :: xstore_glob,ystore_glob,zstore_glob
double precision, dimension(:),allocatable :: xstore_glob,ystore_glob,zstore_glob

! parameters needed to store the radii of the grid points
! in the spherically symmetric Earth
Expand Down
41 changes: 37 additions & 4 deletions src/meshfem3D/setup_MPI_interfaces.f90
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@

subroutine setup_MPI_interfaces(iregion_code)

use meshfem_par, only: &
INCLUDE_CENTRAL_CUBE,myrank,NUMFACES_SHARED
use meshfem_par, only: myrank,INCLUDE_CENTRAL_CUBE,NUMFACES_SHARED

use MPI_interfaces_par

Expand All @@ -52,7 +51,6 @@ subroutine setup_MPI_interfaces(iregion_code)
integer, dimension(:,:),allocatable :: ibool_neighbors
integer :: ier

! allocates temporary arrays for setup routines
! estimates a maximum size of needed arrays
MAX_NEIGHBORS = 8 + NCORNERSCHUNKS
if (INCLUDE_CENTRAL_CUBE) MAX_NEIGHBORS = MAX_NEIGHBORS + NUMMSGS_FACES
Expand Down Expand Up @@ -270,7 +268,7 @@ subroutine setup_MPI_interfaces_cm(MAX_NEIGHBORS,my_neighbors,nibool_neighbors,
do i = 1,num_interfaces_crust_mantle
write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_crust_mantle_proc',myrank, &
'_',my_neighbors_crust_mantle(i)
call write_VTK_data_points(NGLOB_crust_mantle, &
call write_VTK_data_points(NGLOB_CRUST_MANTLE, &
xstore_glob,ystore_glob,zstore_glob, &
ibool_interfaces_crust_mantle(1:nibool_interfaces_crust_mantle(i),i), &
nibool_interfaces_crust_mantle(i),filename)
Expand All @@ -284,6 +282,13 @@ subroutine setup_MPI_interfaces_cm(MAX_NEIGHBORS,my_neighbors,nibool_neighbors,
my_neighbors_crust_mantle,nibool_interfaces_crust_mantle, &
ibool_interfaces_crust_mantle)

! checks cross-rank interface correspondence
call test_MPI_interfaces_matching(IREGION_CRUST_MANTLE, &
num_interfaces_crust_mantle,max_nibool_interfaces_cm, &
my_neighbors_crust_mantle,nibool_interfaces_crust_mantle, &
ibool_interfaces_crust_mantle, &
NGLOB_CRUST_MANTLE,xstore_glob,ystore_glob,zstore_glob)

! checks with assembly of test fields
call test_MPI_cm()

Expand Down Expand Up @@ -424,6 +429,13 @@ subroutine setup_MPI_interfaces_oc(MAX_NEIGHBORS,my_neighbors,nibool_neighbors,
my_neighbors_outer_core,nibool_interfaces_outer_core, &
ibool_interfaces_outer_core)

! checks cross-rank interface correspondence
call test_MPI_interfaces_matching(IREGION_OUTER_CORE, &
num_interfaces_outer_core,max_nibool_interfaces_oc, &
my_neighbors_outer_core,nibool_interfaces_outer_core, &
ibool_interfaces_outer_core, &
NGLOB_OUTER_CORE,xstore_glob,ystore_glob,zstore_glob)

! checks with assembly of test fields
call test_MPI_oc()

Expand Down Expand Up @@ -627,6 +639,13 @@ subroutine setup_MPI_interfaces_ic(MAX_NEIGHBORS,my_neighbors,nibool_neighbors,
my_neighbors_inner_core,nibool_interfaces_inner_core, &
ibool_interfaces_inner_core)

! checks cross-rank interface correspondence
call test_MPI_interfaces_matching(IREGION_INNER_CORE, &
num_interfaces_inner_core,max_nibool_interfaces_ic, &
my_neighbors_inner_core,nibool_interfaces_inner_core, &
ibool_interfaces_inner_core, &
NGLOB_INNER_CORE,xstore_glob,ystore_glob,zstore_glob)

! checks with assembly of test fields
call test_MPI_ic()

Expand Down Expand Up @@ -774,6 +793,13 @@ subroutine setup_MPI_interfaces_trinf(MAX_NEIGHBORS,my_neighbors,nibool_neighbor
my_neighbors_trinfinite,nibool_interfaces_trinfinite, &
ibool_interfaces_trinfinite)

! checks cross-rank interface correspondence
call test_MPI_interfaces_matching(IREGION_TRINFINITE, &
num_interfaces_trinfinite,max_nibool_interfaces_trinfinite, &
my_neighbors_trinfinite,nibool_interfaces_trinfinite, &
ibool_interfaces_trinfinite, &
NGLOB_TRINFINITE,xstore_glob,ystore_glob,zstore_glob)

! checks with assembly of test fields
call test_MPI_trinf()

Expand Down Expand Up @@ -914,6 +940,13 @@ subroutine setup_MPI_interfaces_inf(MAX_NEIGHBORS,my_neighbors,nibool_neighbors,
my_neighbors_infinite,nibool_interfaces_infinite, &
ibool_interfaces_infinite)

! checks cross-rank interface correspondence
call test_MPI_interfaces_matching(IREGION_INFINITE, &
num_interfaces_infinite,max_nibool_interfaces_infinite, &
my_neighbors_infinite,nibool_interfaces_infinite, &
ibool_interfaces_infinite, &
NGLOB_INFINITE,xstore_glob,ystore_glob,zstore_glob)

! checks with assembly of test fields
call test_MPI_inf()

Expand Down
20 changes: 10 additions & 10 deletions src/meshfem3D/setup_inner_outer.f90
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ subroutine setup_inner_outer(iregion_code)
if (DEBUG) then
write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_crust_mantle_proc',myrank
call write_VTK_data_elem_l(NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE, &
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
endif

!debug
Expand Down Expand Up @@ -168,8 +168,8 @@ subroutine setup_inner_outer(iregion_code)
if (DEBUG) then
write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_outer_core_proc',myrank
call write_VTK_data_elem_l(NSPEC_OUTER_CORE,NGLOB_OUTER_CORE, &
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
endif

case (IREGION_INNER_CORE)
Expand Down Expand Up @@ -265,8 +265,8 @@ subroutine setup_inner_outer(iregion_code)
if (DEBUG) then
write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_inner_core_proc',myrank
call write_VTK_data_elem_l(NSPEC_INNER_CORE,NGLOB_INNER_CORE, &
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
endif

case (IREGION_TRINFINITE)
Expand Down Expand Up @@ -311,8 +311,8 @@ subroutine setup_inner_outer(iregion_code)
if (DEBUG) then
write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_trinfinite_proc',myrank
call write_VTK_data_elem_l(NSPEC_TRINFINITE,NGLOB_TRINFINITE, &
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
endif

case (IREGION_INFINITE)
Expand Down Expand Up @@ -357,8 +357,8 @@ subroutine setup_inner_outer(iregion_code)
if (DEBUG) then
write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_innerouter_infinite_proc',myrank
call write_VTK_data_elem_l(NSPEC_INFINITE,NGLOB_INFINITE, &
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
xstore_glob,ystore_glob,zstore_glob, &
ibool,is_on_a_slice_edge,filename)
endif

end select
Expand Down
Loading
Loading