16#ifndef CABANA_HALO_HPP
17#define CABANA_HALO_HPP
23#include <Kokkos_Core.hpp>
24#include <Kokkos_Profiling_ScopedRegion.hpp>
58template <
class MemorySpace,
class BuildType = Export>
102 template <
class IdViewType,
class RankViewType,
typename T = BuildType,
103 std::enable_if_t<std::is_same<T, Export>::value,
int> = 0>
105 const IdViewType& element_ids,
const RankViewType& element_ranks,
106 const std::vector<int>& neighbor_ranks )
108 , _num_local( num_local )
110 if ( element_ids.size() != element_ranks.size() )
111 throw std::runtime_error(
"Cabana::Halo (export): ids and ranks "
112 "views are different sizes!" );
114 auto neighbor_ids = this->createWithTopology(
115 BuildType(), element_ranks, neighbor_ranks );
151 template <
class IdViewType,
class RankViewType,
typename T = BuildType,
152 std::enable_if_t<std::is_same<T, Export>::value,
int> = 0>
154 const IdViewType& element_ids,
const RankViewType& element_ranks )
156 , _num_local( num_local )
158 if ( element_ids.size() != element_ranks.size() )
159 throw std::runtime_error(
"Cabana::Halo (export): ids and ranks "
160 "views are different sizes!" );
163 this->createWithoutTopology( BuildType(), element_ranks );
207 template <
class IdViewType,
class RankViewType,
typename T = BuildType,
208 std::enable_if_t<std::is_same<T, Import>::value,
int> = 0>
210 const IdViewType& element_ids,
const RankViewType& element_ranks,
211 const std::vector<int>& neighbor_ranks )
213 , _num_local( num_local )
215 if ( element_ids.size() != element_ranks.size() )
216 throw std::runtime_error(
"Cabana::Halo (import): ids and ranks "
217 "views are different sizes!" );
219 auto neighbor_ids_ranks_indices = this->createWithTopology(
220 BuildType(), element_ranks, element_ids, neighbor_ranks );
222 std::get<1>( neighbor_ids_ranks_indices ),
223 std::get<2>( neighbor_ids_ranks_indices ) );
257 template <
class IdViewType,
class RankViewType,
typename T = BuildType,
258 std::enable_if_t<std::is_same<T, Import>::value,
int> = 0>
260 const IdViewType& element_ids,
const RankViewType& element_ranks )
262 , _num_local( num_local )
264 if ( element_ids.size() != element_ranks.size() )
265 throw std::runtime_error(
"Cabana::Halo (import): ids and ranks "
266 "views are different sizes!" );
268 auto neighbor_ids_ranks_indices = this->createWithoutTopology(
269 BuildType(), element_ranks, element_ids );
271 std::get<1>( neighbor_ids_ranks_indices ),
272 std::get<2>( neighbor_ids_ranks_indices ) );
280 std::size_t
numLocal()
const {
return _num_local; }
292 std::size_t _num_local;
298struct is_halo_impl :
public std::false_type
302template <
typename MemorySpace,
typename BuildType>
303struct is_halo_impl<
Halo<MemorySpace, BuildType>> :
public std::true_type
310struct is_halo :
public is_halo_impl<typename std::remove_cv<T>::type>::type
323template <
class Halo,
class ParticleData>
325 const Halo& halo,
const ParticleData& particles,
332template <
class HaloType,
class AoSoAType,
class SFINAE =
void>
347template <
class HaloType,
class AoSoAType>
349 typename std::enable_if<is_aosoa<AoSoAType>::value>::type>
381 Gather( HaloType halo, AoSoAType aosoa,
const double overallocation = 1.0 )
382 :
base_type( halo, aosoa, overallocation )
395 template <
class ExecutionSpace>
398 Kokkos::Profiling::ScopedRegion region(
"Cabana::Gather::apply" );
406 auto steering = _halo.getExportSteering();
408 auto gather_send_buffer_func = KOKKOS_LAMBDA(
const std::size_t i )
410 send_buffer( i ) = aosoa.getTuple( steering( i ) );
412 Kokkos::RangePolicy<ExecutionSpace> send_policy( 0, _send_size );
413 Kokkos::parallel_for(
"Cabana::Gather::gather_send_buffer", send_policy,
414 gather_send_buffer_func );
418 const int mpi_tag = 2345;
421 int num_n = _halo.numNeighbor();
422 std::vector<MPI_Request> requests( num_n );
423 std::pair<std::size_t, std::size_t> recv_range = { 0, 0 };
424 for (
int n = 0; n < num_n; ++n )
426 recv_range.second = recv_range.first + _halo.numImport( n );
428 auto recv_subview = Kokkos::subview( recv_buffer, recv_range );
430 MPI_Irecv( recv_subview.data(),
431 recv_subview.size() *
sizeof(
data_type ), MPI_BYTE,
432 _halo.neighborRank( n ), mpi_tag, _halo.comm(),
435 recv_range.first = recv_range.second;
439 std::pair<std::size_t, std::size_t> send_range = { 0, 0 };
440 for (
int n = 0; n < num_n; ++n )
442 send_range.second = send_range.first + _halo.numExport( n );
444 auto send_subview = Kokkos::subview( send_buffer, send_range );
446 MPI_Send( send_subview.data(),
447 send_subview.size() *
sizeof(
data_type ), MPI_BYTE,
448 _halo.neighborRank( n ), mpi_tag, _halo.comm() );
450 send_range.first = send_range.second;
454 std::vector<MPI_Status> status( num_n );
456 MPI_Waitall( requests.size(), requests.data(), status.data() );
457 if ( MPI_SUCCESS != ec )
458 throw std::logic_error(
459 "Cabana::Gather::apply (AoSoA): Failed MPI Communication" );
462 std::size_t num_local = _halo.numLocal();
463 auto extract_recv_buffer_func = KOKKOS_LAMBDA(
const std::size_t i )
465 std::size_t ghost_idx = i + num_local;
466 aosoa.setTuple( ghost_idx, recv_buffer( i ) );
468 Kokkos::RangePolicy<ExecutionSpace> recv_policy( 0, _recv_size );
469 Kokkos::parallel_for(
"Cabana::Gather::apply::extract_recv_buffer",
470 recv_policy, extract_recv_buffer_func );
474 MPI_Barrier( _halo.comm() );
485 void reserve(
const HaloType& halo, AoSoAType& aosoa )
488 throw std::runtime_error(
489 "Cabana::Gather:reserve: "
490 "AoSoA is the wrong size for gather! (Label: " +
491 aosoa.label() +
")" );
503 void reserve(
const HaloType& halo, AoSoAType& aosoa,
504 const double overallocation )
507 throw std::runtime_error(
508 "Cabana::Gather:reserve: "
509 "AoSoA is the wrong size for gather! (Label: " +
510 aosoa.label() +
")" );
517 plan_type _halo = base_type::_comm_plan;
518 using base_type::_recv_size;
519 using base_type::_send_size;
533template <
class HaloType,
class SliceType>
535 typename std::enable_if<is_slice<SliceType>::value>::type>
567 Gather( HaloType halo, SliceType
slice,
const double overallocation = 1.0 )
581 template <
class ExecutionSpace>
584 Kokkos::Profiling::ScopedRegion region(
"Cabana::Gather::apply" );
595 auto slice_data =
slice.data();
598 auto steering = _halo.getExportSteering();
601 auto gather_send_buffer_func = KOKKOS_LAMBDA(
const std::size_t i )
603 auto s = SliceType::index_type::s( steering( i ) );
604 auto a = SliceType::index_type::a( steering( i ) );
605 std::size_t slice_offset = s *
slice.stride( 0 ) + a;
606 for ( std::size_t n = 0; n < num_comp; ++n )
607 send_buffer( i, n ) =
608 slice_data[slice_offset + n * SliceType::vector_length];
610 Kokkos::RangePolicy<ExecutionSpace> send_policy( 0, _send_size );
611 Kokkos::parallel_for(
"Cabana::Gather::gather_send_buffer", send_policy,
612 gather_send_buffer_func );
616 const int mpi_tag = 2345;
619 int num_n = _halo.numNeighbor();
620 std::vector<MPI_Request> requests( num_n );
621 std::pair<std::size_t, std::size_t> recv_range = { 0, 0 };
622 for (
int n = 0; n < num_n; ++n )
624 recv_range.second = recv_range.first + _halo.numImport( n );
627 Kokkos::subview( recv_buffer, recv_range, Kokkos::ALL );
629 MPI_Irecv( recv_subview.data(),
630 recv_subview.size() *
sizeof(
data_type ), MPI_BYTE,
631 _halo.neighborRank( n ), mpi_tag, _halo.comm(),
634 recv_range.first = recv_range.second;
638 std::pair<std::size_t, std::size_t> send_range = { 0, 0 };
639 for (
int n = 0; n < num_n; ++n )
641 send_range.second = send_range.first + _halo.numExport( n );
644 Kokkos::subview( send_buffer, send_range, Kokkos::ALL );
646 MPI_Send( send_subview.data(),
647 send_subview.size() *
sizeof(
data_type ), MPI_BYTE,
648 _halo.neighborRank( n ), mpi_tag, _halo.comm() );
650 send_range.first = send_range.second;
654 std::vector<MPI_Status> status( num_n );
656 MPI_Waitall( requests.size(), requests.data(), status.data() );
657 if ( MPI_SUCCESS != ec )
658 throw std::logic_error(
659 "Cabana::Gather::apply (Slice): Failed MPI Communication" );
662 std::size_t num_local = _halo.numLocal();
663 auto extract_recv_buffer_func = KOKKOS_LAMBDA(
const std::size_t i )
665 std::size_t ghost_idx = i + num_local;
666 auto s = SliceType::index_type::s( ghost_idx );
667 auto a = SliceType::index_type::a( ghost_idx );
668 std::size_t slice_offset = s *
slice.stride( 0 ) + a;
669 for ( std::size_t n = 0; n < num_comp; ++n )
670 slice_data[slice_offset + SliceType::vector_length * n] =
673 Kokkos::RangePolicy<ExecutionSpace> recv_policy( 0, _recv_size );
674 Kokkos::parallel_for(
"Cabana::Gather::extract_recv_buffer",
675 recv_policy, extract_recv_buffer_func );
679 MPI_Barrier( _halo.comm() );
693 const double overallocation )
696 throw std::runtime_error(
697 "Cabana::Gather:reserve: "
698 "Slice is the wrong size for gather! (Label: " +
699 slice.label() +
")" );
713 throw std::runtime_error(
714 "Cabana::Gather:reserve: "
715 "Slice is the wrong size for gather! (Label: " +
716 slice.label() +
")" );
722 plan_type _halo = base_type::_comm_plan;
723 using base_type::_recv_size;
724 using base_type::_send_size;
741template <
class HaloType,
class ParticleDataType>
743 const double overallocation = 1.0 )
766template <
class HaloType,
class ParticleDataType>
767void gather(
const HaloType& halo, ParticleDataType& data )
789template <
class HaloType,
class SliceType>
822 Scatter( HaloType halo, SliceType
slice,
const double overallocation = 1.0 )
836 template <
class ExecutionSpace>
839 Kokkos::Profiling::ScopedRegion region(
"Cabana::Scatter::apply" );
852 Kokkos::MemoryTraits<Kokkos::Unmanaged>>
856 std::size_t num_local = _halo.numLocal();
857 auto extract_send_buffer_func = KOKKOS_LAMBDA(
const std::size_t i )
859 std::size_t ghost_idx = i + num_local;
860 auto s = SliceType::index_type::s( ghost_idx );
861 auto a = SliceType::index_type::a( ghost_idx );
862 std::size_t slice_offset = s *
slice.stride( 0 ) + a;
863 for ( std::size_t n = 0; n < num_comp; ++n )
864 send_buffer( i, n ) =
865 slice_data( slice_offset + SliceType::vector_length * n );
867 Kokkos::RangePolicy<ExecutionSpace> send_policy( 0, _send_size );
868 Kokkos::parallel_for(
"Cabana::Scatter::apply::extract_send_buffer",
869 send_policy, extract_send_buffer_func );
873 const int mpi_tag = 2345;
876 int num_n = _halo.numNeighbor();
877 std::vector<MPI_Request> requests( num_n );
878 std::pair<std::size_t, std::size_t> recv_range = { 0, 0 };
879 for (
int n = 0; n < num_n; ++n )
881 recv_range.second = recv_range.first + _halo.numExport( n );
884 Kokkos::subview( recv_buffer, recv_range, Kokkos::ALL );
886 MPI_Irecv( recv_subview.data(),
887 recv_subview.size() *
sizeof(
data_type ), MPI_BYTE,
888 _halo.neighborRank( n ), mpi_tag, _halo.comm(),
891 recv_range.first = recv_range.second;
895 std::pair<std::size_t, std::size_t> send_range = { 0, 0 };
896 for (
int n = 0; n < num_n; ++n )
898 send_range.second = send_range.first + _halo.numImport( n );
901 Kokkos::subview( send_buffer, send_range, Kokkos::ALL );
903 MPI_Send( send_subview.data(),
904 send_subview.size() *
sizeof(
data_type ), MPI_BYTE,
905 _halo.neighborRank( n ), mpi_tag, _halo.comm() );
907 send_range.first = send_range.second;
911 std::vector<MPI_Status> status( num_n );
913 MPI_Waitall( requests.size(), requests.data(), status.data() );
914 if ( MPI_SUCCESS != ec )
915 throw std::logic_error(
"Cabana::Scatter::apply (Slice): "
916 "Failed MPI Communication" );
919 auto steering = _halo.getExportSteering();
922 auto scatter_recv_buffer_func = KOKKOS_LAMBDA(
const std::size_t i )
924 auto s = SliceType::index_type::s( steering( i ) );
925 auto a = SliceType::index_type::a( steering( i ) );
926 std::size_t slice_offset = s *
slice.stride( 0 ) + a;
927 for ( std::size_t n = 0; n < num_comp; ++n )
929 &slice_data( slice_offset + SliceType::vector_length * n ),
930 recv_buffer( i, n ) );
932 Kokkos::RangePolicy<ExecutionSpace> recv_policy( 0, _recv_size );
933 Kokkos::parallel_for(
"Cabana::Scatter::apply::scatter_recv_buffer",
934 recv_policy, scatter_recv_buffer_func );
938 MPI_Barrier( _halo.comm() );
953 const double overallocation )
956 throw std::runtime_error(
957 "Cabana::Scatter::reserve: "
958 "Slice is the wrong size for scatter! (Label: " +
959 slice.label() +
")" );
973 throw std::runtime_error(
974 "Cabana::Scatter::reserve: "
975 "Slice is the wrong size for scatter! (Label: " +
976 slice.label() +
")" );
1000template <
class HaloType,
class SliceType>
1002 const double overallocation = 1.0,
1028template <
class HaloType,
class SliceType>
Array-of-Struct-of-Arrays particle data structure.
Multi-node communication patterns.
Slice a single particle property from an AoSoA.
typename plan_type::execution_space execution_space
Definition Cabana_CommunicationPlan.hpp:1742
buffer_type getSendBuffer() const
Definition Cabana_CommunicationPlan.hpp:1772
particle_data_type getData() const
Definition Cabana_CommunicationPlan.hpp:1777
plan_type _comm_plan
Definition Cabana_CommunicationPlan.hpp:1870
std::size_t _recv_size
Definition Cabana_CommunicationPlan.hpp:1878
typename comm_data_type::data_type data_type
Definition Cabana_CommunicationPlan.hpp:1752
typename comm_data_type::buffer_type buffer_type
Definition Cabana_CommunicationPlan.hpp:1754
auto getSliceComponents()
Definition Cabana_CommunicationPlan.hpp:1867
HaloType plan_type
Definition Cabana_CommunicationPlan.hpp:1740
CommunicationData(const HaloType &comm_plan, const particle_data_type &particles, const double overallocation=1.0)
Definition Cabana_CommunicationPlan.hpp:1762
typename comm_data_type::memory_space memory_space
Definition Cabana_CommunicationPlan.hpp:1750
buffer_type getReceiveBuffer() const
Definition Cabana_CommunicationPlan.hpp:1774
std::size_t _send_size
Definition Cabana_CommunicationPlan.hpp:1876
void createExportSteering(const PackViewType &neighbor_ids, const RankViewType &element_export_ranks)
Create the export steering vector.
Definition Cabana_CommunicationPlan.hpp:1504
MPI_Comm comm() const
Get the MPI communicator.
Definition Cabana_CommunicationPlan.hpp:480
std::size_t totalNumImport() const
Get the total number of imports this rank will do.
Definition Cabana_CommunicationPlan.hpp:537
CommunicationPlan(MPI_Comm comm)
Constructor.
Definition Cabana_CommunicationPlan.hpp:458
typename base_type::memory_space memory_space
Kokkos memory space.
Definition Cabana_Halo.hpp:363
void apply() override
Perform the communication (migrate, gather, scatter).
Definition Cabana_Halo.hpp:477
void reserve(const HaloType &halo, AoSoAType &aosoa, const double overallocation)
Reserve new buffers as needed and update the halo and AoSoA data.
Definition Cabana_Halo.hpp:503
auto totalSend()
Total gather send size for this rank.
Definition Cabana_Halo.hpp:388
typename base_type::data_type data_type
Communication data type.
Definition Cabana_Halo.hpp:365
Gather(HaloType halo, AoSoAType aosoa, const double overallocation=1.0)
Definition Cabana_Halo.hpp:381
auto totalReceive()
Total gather receive size for this rank.
Definition Cabana_Halo.hpp:390
void apply(ExecutionSpace)
Perform the gather operation.
Definition Cabana_Halo.hpp:396
typename base_type::buffer_type buffer_type
Communication buffer type.
Definition Cabana_Halo.hpp:367
typename base_type::plan_type plan_type
Communication plan type (Halo)
Definition Cabana_Halo.hpp:359
typename base_type::execution_space execution_space
Kokkos execution space.
Definition Cabana_Halo.hpp:361
void reserve(const HaloType &halo, AoSoAType &aosoa)
Reserve new buffers as needed and update the halo and AoSoA data.
Definition Cabana_Halo.hpp:485
CommunicationData< HaloType, CommunicationDataAoSoA< AoSoAType > > base_type
Base type.
Definition Cabana_Halo.hpp:356
typename base_type::memory_space memory_space
Kokkos memory space.
Definition Cabana_Halo.hpp:549
typename base_type::buffer_type buffer_type
Communication buffer type.
Definition Cabana_Halo.hpp:553
CommunicationData< HaloType, CommunicationDataSlice< SliceType > > base_type
Base type.
Definition Cabana_Halo.hpp:542
void apply(ExecutionSpace)
Perform the gather operation.
Definition Cabana_Halo.hpp:582
void apply() override
Perform the communication (migrate, gather, scatter).
Definition Cabana_Halo.hpp:682
auto totalReceive()
Total gather receive size for this rank.
Definition Cabana_Halo.hpp:576
Gather(HaloType halo, SliceType slice, const double overallocation=1.0)
Definition Cabana_Halo.hpp:567
void reserve(const HaloType &halo, const SliceType &slice, const double overallocation)
Reserve new buffers as needed and update the halo and slice data.
Definition Cabana_Halo.hpp:692
typename base_type::execution_space execution_space
Kokkos execution space.
Definition Cabana_Halo.hpp:547
typename base_type::plan_type plan_type
Communication plan type (Halo)
Definition Cabana_Halo.hpp:545
auto totalSend()
Total gather send size for this rank.
Definition Cabana_Halo.hpp:574
void reserve(const HaloType &halo, const SliceType &slice)
Reserve new buffers as needed and update the halo and slice data.
Definition Cabana_Halo.hpp:710
typename base_type::data_type data_type
Communication data type.
Definition Cabana_Halo.hpp:551
Definition Cabana_Halo.hpp:333
A communication plan for scattering and gathering of ghosted data.
Definition Cabana_Halo.hpp:60
std::size_t numGhost() const
Get the number of ghost elements this rank. Use this to resize a data structure for scatter/gather op...
Definition Cabana_Halo.hpp:289
Halo(MPI_Comm comm, const std::size_t num_local, const IdViewType &element_ids, const RankViewType &element_ranks, const std::vector< int > &neighbor_ranks)
Neighbor and export rank constructor. Use this when you don't know who you will receiving from - only...
Definition Cabana_Halo.hpp:104
std::size_t numLocal() const
Get the number of elements locally owned by this rank.
Definition Cabana_Halo.hpp:280
Halo(MPI_Comm comm, const std::size_t num_local, const IdViewType &element_ids, const RankViewType &element_ranks)
Export rank constructor. Use this when you don't know who you will receiving from - only who you are ...
Definition Cabana_Halo.hpp:153
Synchronously scatter data from the ghosts to the local decomposition of a slice using the halo rever...
Definition Cabana_Halo.hpp:792
void reserve(const HaloType &halo, const SliceType &slice)
Reserve new buffers as needed and update the halo and slice data.
Definition Cabana_Halo.hpp:970
CommunicationData< HaloType, CommunicationDataSlice< SliceType > > base_type
Base type.
Definition Cabana_Halo.hpp:797
typename base_type::memory_space memory_space
Kokkos memory space.
Definition Cabana_Halo.hpp:804
void apply(ExecutionSpace)
Perform the scatter operation.
Definition Cabana_Halo.hpp:837
typename base_type::data_type data_type
Communication data type.
Definition Cabana_Halo.hpp:806
void reserve(const HaloType &halo, const SliceType &slice, const double overallocation)
Reserve new buffers as needed and update the halo and slice data. Reallocation only occurs if there i...
Definition Cabana_Halo.hpp:952
typename base_type::plan_type plan_type
Communication plan type (Halo).
Definition Cabana_Halo.hpp:800
auto totalReceive()
Total scatter receive size for this rank.
Definition Cabana_Halo.hpp:831
void apply() override
Perform the communication (migrate, gather, scatter).
Definition Cabana_Halo.hpp:941
typename base_type::buffer_type buffer_type
Communication buffer type.
Definition Cabana_Halo.hpp:808
typename base_type::execution_space execution_space
Kokkos execution space.
Definition Cabana_Halo.hpp:802
Scatter(HaloType halo, SliceType slice, const double overallocation=1.0)
Definition Cabana_Halo.hpp:822
auto totalSend()
Total scatter send size for this rank.
Definition Cabana_Halo.hpp:829
Core: particle data structures and algorithms.
Definition Cabana_AoSoA.hpp:36
void gather(const HaloType &halo, ParticleDataType &data)
Synchronously gather data from the local decomposition to the ghosts using the halo forward communica...
Definition Cabana_Halo.hpp:767
AoSoA_t::template member_slice_type< M > slice(const AoSoA_t &aosoa, const std::string &slice_label="")
Create a slice from an AoSoA.
Definition Cabana_AoSoA.hpp:77
void scatter(const HaloType &halo, SliceType &slice, typename std::enable_if<(is_halo< HaloType >::value &&is_slice< SliceType >::value), int >::type *=0)
Synchronously scatter data from the ghosts to the local decomposition of a slice using the halo rever...
Definition Cabana_Halo.hpp:1029
auto createScatter(const HaloType &halo, const SliceType &slice, const double overallocation=1.0, typename std::enable_if<(is_halo< HaloType >::value &&is_slice< SliceType >::value), int >::type *=0)
Create the scatter.
Definition Cabana_Halo.hpp:1001
auto createGather(const HaloType &halo, const ParticleDataType &data, const double overallocation=1.0)
Create the gather.
Definition Cabana_Halo.hpp:742
bool haloCheckValidSize(const Halo &halo, const ParticleData &particles, typename std::enable_if<(is_halo< Halo >::value), int >::type *=0)
Ensure the particle size matches the total halo (local and ghost) size.
Definition Cabana_Halo.hpp:324
Halo static type checker.
Definition Cabana_Halo.hpp:311
Slice static type checker.
Definition Cabana_Slice.hpp:868