Communication plan base class. More...

#include <Cabana_CommunicationPlan.hpp>

Inheritance diagram for Cabana::CommunicationPlan< MemorySpace >:

[legend]

Public Types
using	memory_space = MemorySpace
	Kokkos memory space.

using	execution_space = typename memory_space::execution_space
	Default execution space.

using	size_type = typename memory_space::memory_space::size_type
	Size type.

Public Member Functions
	CommunicationPlan (MPI_Comm comm)
	Constructor.

MPI_Comm	comm () const
	Get the MPI communicator.

int	numNeighbor () const
	Get the number of neighbor ranks that this rank will communicate with.

int	neighborRank (const int neighbor) const
	Given a local neighbor id get its rank in the MPI communicator.

std::size_t	numExport (const int neighbor) const
	Get the number of elements this rank will export to a given neighbor.

std::size_t	totalNumExport () const
	Get the total number of exports this rank will do.

std::size_t	numImport (const int neighbor) const
	Get the number of elements this rank will import from a given neighbor.

std::size_t	totalNumImport () const
	Get the total number of imports this rank will do.

std::size_t	exportSize () const
	Get the number of export elements.

Kokkos::View< std::size_t *, memory_space >	getExportSteering () const
	Get the steering vector for the exports.

template<class ExecutionSpace, class RankViewType>
Kokkos::View< size_type *, memory_space >	createWithTopology (ExecutionSpace exec_space, Export, const RankViewType &element_export_ranks, const std::vector< int > &neighbor_ranks)
	Neighbor and export rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

template<class RankViewType>
Kokkos::View< size_type *, memory_space >	createWithTopology (Export, const RankViewType &element_export_ranks, const std::vector< int > &neighbor_ranks)
	Neighbor and export rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

template<class ExecutionSpace, class RankViewType>
Kokkos::View< size_type *, memory_space >	createWithoutTopology (ExecutionSpace exec_space, Export, const RankViewType &element_export_ranks)
	Export rank creator. Use this when you don't know who you will receiving from - only who you are sending to. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

template<class RankViewType>
Kokkos::View< size_type *, memory_space >	createWithoutTopology (Export, const RankViewType &element_export_ranks)
	Export rank creator. Use this when you don't know who you will receiving from - only who you are sending to. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

template<class ExecutionSpace, class RankViewType, class IdViewType>
auto	createWithTopology (ExecutionSpace exec_space, Import, const RankViewType &element_import_ranks, const IdViewType &element_import_ids, const std::vector< int > &neighbor_ranks) -> std::tuple< Kokkos::View< typename RankViewType::size_type , typename RankViewType::memory_space >, Kokkos::View< int , typename RankViewType::memory_space >, Kokkos::View< int *, typename IdViewType::memory_space > >
	Neighbor and import rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

template<class RankViewType, class IdViewType>
auto	createWithTopology (Import, const RankViewType &element_import_ranks, const IdViewType &element_import_ids, const std::vector< int > &neighbor_ranks)
	Neighbor and import rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

template<class ExecutionSpace, class RankViewType, class IdViewType>
auto	createWithoutTopology (ExecutionSpace exec_space, Import, const RankViewType &element_import_ranks, const IdViewType &element_import_ids) -> std::tuple< Kokkos::View< typename RankViewType::size_type , typename RankViewType::memory_space >, Kokkos::View< int , typename RankViewType::memory_space >, Kokkos::View< int *, typename IdViewType::memory_space > >
	Import rank creator. Use this when you don't know who you will be receiving from - only who you are importing from. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

template<class RankViewType, class IdViewType>
auto	createWithoutTopology (Import, const RankViewType &element_import_ranks, const IdViewType &element_import_ids)
	Import rank creator. Use this when you don't know who you will be receiving from - only who you are importing from. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

template<class PackViewType, class RankViewType>
void	createExportSteering (const PackViewType &neighbor_ids, const RankViewType &element_export_ranks)
	Create the export steering vector.

template<class PackViewType, class RankViewType, class IdViewType>
void	createExportSteering (const PackViewType &neighbor_ids, const RankViewType &element_export_ranks, const IdViewType &element_export_ids)
	Create the export steering vector.

Detailed Description

template<class MemorySpace>
class Cabana::CommunicationPlan< MemorySpace >

Communication plan base class.

Template Parameters

DeviceType Device type for which the data for this class will be allocated and where parallel execution will occur.

The communication plan computes how to redistribute elements in a parallel data structure using MPI. Given a list of data elements on the local MPI rank and their destination ranks, the communication plan computes which rank each process is sending and receiving from and how many elements we will send and receive. In addition, it provides an export steering vector which describes how to pack the local data to be exported into contiguous send buffers for each destination rank (in the forward communication plan).

Some nomenclature:

Export - elements we are sending in the forward communication plan.

Import - elements we are receiving in the forward communication plan.

Note: If a communication plan does self-sends (i.e. exports and imports data from its own ranks) then this data is first in the data structure. What this means is that neighbor 0 is the local rank and the data for that rank that is being exported will appear first in the steering vector.

Constructor & Destructor Documentation

◆ CommunicationPlan()

template<class MemorySpace>

Cabana::CommunicationPlan< MemorySpace >::CommunicationPlan ( MPI_Comm comm )

inline

Constructor.

Parameters

comm	The MPI communicator over which the distributor is defined.

Member Function Documentation

◆ createExportSteering() [1/2]

template<class MemorySpace>

template<class PackViewType, class RankViewType>

void Cabana::CommunicationPlan< MemorySpace >::createExportSteering	(	const PackViewType &	neighbor_ids,
		const RankViewType &	element_export_ranks )

inline

Create the export steering vector.

Creates an array describing which export element ids are moved to which location in the send buffer of the communication plan. Ordered such that if a rank sends to itself then those values come first.

Parameters

neighbor_ids	The id of each element in the neighbor send buffers.
element_export_ranks	The ranks to which we are exporting each element. We use this to build the steering vector. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.

◆ createExportSteering() [2/2]

template<class MemorySpace>

template<class PackViewType, class RankViewType, class IdViewType>

void Cabana::CommunicationPlan< MemorySpace >::createExportSteering	(	const PackViewType &	neighbor_ids,
		const RankViewType &	element_export_ranks,
		const IdViewType &	element_export_ids )

inline

Create the export steering vector.

Creates an array describing which export element ids are moved to which location in the contiguous send buffer of the communication plan. Ordered such that if a rank sends to itself then those values come first.

Parameters

neighbor_ids	The id of each element in the neighbor send buffers.
element_export_ranks	The ranks to which we are exporting each element. We use this to build the steering vector. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.
element_export_ids	The local ids of the elements to be exported. This corresponds with the export ranks vector and must be the same length if defined. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.

◆ createWithoutTopology() [1/4]

template<class MemorySpace>

template<class ExecutionSpace, class RankViewType>

Kokkos::View< size_type *, memory_space > Cabana::CommunicationPlan< MemorySpace >::createWithoutTopology	(	ExecutionSpace	exec_space,
		Export	,
		const RankViewType &	element_export_ranks )

inline

Export rank creator. Use this when you don't know who you will receiving from - only who you are sending to. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

Parameters

exec_space Kokkos execution space.

element_export_ranks The destination rank in the target decomposition of each locally owned element in the source decomposition. Each element will have one unique destination to which it will be exported. This export rank may any one of the listed neighbor ranks which can include the calling rank. An export rank of -1 will signal that this element is not to be exported and will be ignored in the data migration. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.

Returns: The location of each export element in the send buffer for its given neighbor.

Note: Calling this function completely updates the state of this object and invalidates the previous state.; For elements that you do not wish to export, use an export rank of -1 to signal that this element is not to be exported and will be ignored in the data migration. In other words, this element will be completely* removed in the new decomposition. If the data is staying on this rank, just use this rank as the export destination and the data will be efficiently migrated.

◆ createWithoutTopology() [2/4]

template<class MemorySpace>

template<class ExecutionSpace, class RankViewType, class IdViewType>

auto Cabana::CommunicationPlan< MemorySpace >::createWithoutTopology	(	ExecutionSpace	exec_space,
		Import	,
		const RankViewType &	element_import_ranks,
		const IdViewType &	element_import_ids ) -> std::tuple<Kokkos::View<typename RankViewType::size_type, typename RankViewType::memory_space>, Kokkos::View<int, typename RankViewType::memory_space>, Kokkos::View<int*, typename IdViewType::memory_space>>

inline

Import rank creator. Use this when you don't know who you will be receiving from - only who you are importing from. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

Parameters

exec_space	Kokkos execution space.
element_import_ranks	The source rank in the target decomposition of each remotely owned element in element_import_ids. This import rank may be any one of the listed neighbor ranks which can include the calling rank. The input is expected to be a Kokkos view in the same memory space as the communication plan.
element_import_ids	The local IDs of remotely owned elements that are to be imported. These are local IDs on the remote rank. element_import_ids is mapped such that element_import_ids(i) lives on remote rank element_import_ranks(i).

Returns: A tuple of Kokkos views, where: Element 1: The location of each export element in the send buffer for its given neighbor. Element 2: The remote ranks this rank will export to Element 3: The local IDs this rank will export Elements 2 and 3 are mapped in the same way as element_import_ranks and element_import_ids

Note: Calling this function completely updates the state of this object and invalidates the previous state.; Unlike creating from exports, an import rank of -1 is not supported.

◆ createWithoutTopology() [3/4]

template<class MemorySpace>

template<class RankViewType>

Kokkos::View< size_type *, memory_space > Cabana::CommunicationPlan< MemorySpace >::createWithoutTopology	(	Export	,
		const RankViewType &	element_export_ranks )

inline

Export rank creator. Use this when you don't know who you will receiving from - only who you are sending to. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

Parameters

element_export_ranks The destination rank in the target decomposition of each locally owned element in the source decomposition. Each element will have one unique destination to which it will be exported. This export rank may any one of the listed neighbor ranks which can include the calling rank. An export rank of -1 will signal that this element is not to be exported and will be ignored in the data migration. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.

Returns: The location of each export element in the send buffer for its given neighbor.

Note: Calling this function completely updates the state of this object and invalidates the previous state.; For elements that you do not wish to export, use an export rank of -1 to signal that this element is not to be exported and will be ignored in the data migration. In other words, this element will be completely* removed in the new decomposition. If the data is staying on this rank, just use this rank as the export destination and the data will be efficiently migrated.

◆ createWithoutTopology() [4/4]

template<class MemorySpace>

template<class RankViewType, class IdViewType>

auto Cabana::CommunicationPlan< MemorySpace >::createWithoutTopology	(	Import	,
		const RankViewType &	element_import_ranks,
		const IdViewType &	element_import_ids )

inline

Import rank creator. Use this when you don't know who you will be receiving from - only who you are importing from. This is less efficient than if we already knew who our neighbors were because we have to determine the topology of the point-to-point communication first.

Parameters

element_import_ranks	The source rank in the target decomposition of each remotely owned element in element_import_ids. This import rank may be any one of the listed neighbor ranks which can include the calling rank. The input is expected to be a Kokkos view in the same memory space as the communication plan.
element_import_ids	The local IDs of remotely owned elements that are to be imported. These are local IDs on the remote rank. element_import_ids is mapped such that element_import_ids(i) lives on remote rank element_import_ranks(i).

Returns: A tuple of Kokkos views, where: Element 1: The location of each export element in the send buffer for its given neighbor. Element 2: The remote ranks this rank will export to Element 3: The local IDs this rank will export Elements 2 and 3 are mapped in the same way as element_import_ranks and element_import_ids

Note: Calling this function completely updates the state of this object and invalidates the previous state.; Unlike creating from exports, an import rank of -1 is not supported.

◆ createWithTopology() [1/4]

template<class MemorySpace>

template<class ExecutionSpace, class RankViewType>

Kokkos::View< size_type *, memory_space > Cabana::CommunicationPlan< MemorySpace >::createWithTopology	(	ExecutionSpace	exec_space,
		Export	,
		const RankViewType &	element_export_ranks,
		const std::vector< int > &	neighbor_ranks )

inline

Neighbor and export rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

Parameters

exec_space	Kokkos execution space.
element_export_ranks	The destination rank in the target decomposition of each locally owned element in the source decomposition. Each element will have one unique destination to which it will be exported. This export rank may be any one of the listed neighbor ranks which can include the calling rank. An export rank of -1 will signal that this element is not to be exported and will be ignored in the data migration. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.
neighbor_ranks	List of ranks this rank will send to and receive from. This list can include the calling rank. This is effectively a description of the topology of the point-to-point communication plan. Only the unique elements in this list are used.

Returns: The location of each export element in the send buffer for its given neighbor.

Note: Calling this function completely updates the state of this object and invalidates the previous state.; For elements that you do not wish to export, use an export rank of -1 to signal that this element is not to be exported and will be ignored in the data migration. In other words, this element will be completely* removed in the new decomposition. If the data is staying on this rank, just use this rank as the export destination and the data will be efficiently migrated.

◆ createWithTopology() [2/4]

template<class MemorySpace>

template<class ExecutionSpace, class RankViewType, class IdViewType>

auto Cabana::CommunicationPlan< MemorySpace >::createWithTopology	(	ExecutionSpace	exec_space,
		Import	,
		const RankViewType &	element_import_ranks,
		const IdViewType &	element_import_ids,
		const std::vector< int > &	neighbor_ranks ) -> std::tuple<Kokkos::View<typename RankViewType::size_type, typename RankViewType::memory_space>, Kokkos::View<int, typename RankViewType::memory_space>, Kokkos::View<int*, typename IdViewType::memory_space>>

inline

Neighbor and import rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

Parameters

exec_space	Kokkos execution space.
element_import_ranks	The source rank in the target decomposition of each remotely owned element in element_import_ids. This import rank may be any one of the listed neighbor ranks which can include the calling rank. The input is expected to be a Kokkos view in the same memory space as the communication plan.
element_import_ids	The local IDs of remotely owned elements that are to be imported. These are local IDs on the remote rank. element_import_ids is mapped such that element_import_ids(i) lives on remote rank element_import_ranks(i).
neighbor_ranks	List of ranks this rank will send to and receive from. This list can include the calling rank. This is effectively a description of the topology of the point-to-point communication plan. Only the unique elements in this list are used.

Returns: A tuple of Kokkos views, where: Element 1: The location of each export element in the send buffer for its given neighbor. Element 2: The remote ranks this rank will export to Element 3: The local IDs this rank will export Elements 2 and 3 are mapped in the same way as element_import_ranks and element_import_ids

Note: Calling this function completely updates the state of this object and invalidates the previous state.; Unlike creating from exports, an import rank of -1 is not supported.

◆ createWithTopology() [3/4]

template<class MemorySpace>

template<class RankViewType>

Kokkos::View< size_type *, memory_space > Cabana::CommunicationPlan< MemorySpace >::createWithTopology	(	Export	,
		const RankViewType &	element_export_ranks,
		const std::vector< int > &	neighbor_ranks )

inline

Neighbor and export rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

Parameters

element_export_ranks The destination rank in the target decomposition of each locally owned element in the source decomposition. Each element will have one unique destination to which it will be exported. This export rank may be any one of the listed neighbor ranks which can include the calling rank. An export rank of -1 will signal that this element is not to be exported and will be ignored in the data migration. The input is expected to be a Kokkos view or Cabana slice in the same memory space as the communication plan.

neighbor_ranks List of ranks this rank will send to and receive from. This list can include the calling rank. This is effectively a description of the topology of the point-to-point communication plan. Only the unique elements in this list are used.

Returns: The location of each export element in the send buffer for its given neighbor.

Note: Calling this function completely updates the state of this object and invalidates the previous state.; For elements that you do not wish to export, use an export rank of -1 to signal that this element is not to be exported and will be ignored in the data migration. In other words, this element will be completely* removed in the new decomposition. If the data is staying on this rank, just use this rank as the export destination and the data will be efficiently migrated.

◆ createWithTopology() [4/4]

template<class MemorySpace>

template<class RankViewType, class IdViewType>

auto Cabana::CommunicationPlan< MemorySpace >::createWithTopology	(	Import	,
		const RankViewType &	element_import_ranks,
		const IdViewType &	element_import_ids,
		const std::vector< int > &	neighbor_ranks )

inline

Neighbor and import rank creator. Use this when you already know which ranks neighbor each other (i.e. every rank already knows who they will be sending and receiving from) as it will be more efficient. In this case you already know the topology of the point-to-point communication but not how much data to send to and receive from the neighbors.

Parameters

element_import_ranks	The source rank in the target decomposition of each remotely owned element in element_import_ids. This import rank may be any one of the listed neighbor ranks which can include the calling rank. The input is expected to be a Kokkos view in the same memory space as the communication plan.
element_import_ids	The local IDs of remotely owned elements that are to be imported. These are local IDs on the remote rank. element_import_ids is mapped such that element_import_ids(i) lives on remote rank element_import_ranks(i).
neighbor_ranks	List of ranks this rank will send to and receive from. This list can include the calling rank. This is effectively a description of the topology of the point-to-point communication plan. Only the unique elements in this list are used.

Returns: A tuple of Kokkos views, where: Element 1: The location of each export element in the send buffer for its given neighbor. Element 2: The remote ranks this rank will export to Element 3: The local IDs this rank will export Elements 2 and 3 are mapped in the same way as element_import_ranks and element_import_ids

Note: Calling this function completely updates the state of this object and invalidates the previous state.; Unlike creating from exports, an import rank of -1 is not supported.

◆ exportSize()

template<class MemorySpace>

std::size_t Cabana::CommunicationPlan< MemorySpace >::exportSize ( ) const

inline

Get the number of export elements.

Returns: The number of export elements.

Whenever the communication plan is applied, this is the total number of elements expected to be input on the sending ranks (in the forward communication plan). This will be different than the number returned by totalNumExport() if some of the export ranks used in the construction are -1 and therefore will not particpate in an export operation.

◆ getExportSteering()

template<class MemorySpace>

Kokkos::View< std::size_t *, memory_space > Cabana::CommunicationPlan< MemorySpace >::getExportSteering ( ) const

inline

Get the steering vector for the exports.

Returns: The steering vector for the exports.

The steering vector places exports in contiguous chunks by destination rank. The chunks are in consecutive order based on the local neighbor id (i.e. all elements going to neighbor with local id 0 first, then all elements going to neighbor with local id 1, etc.).

◆ neighborRank()

template<class MemorySpace>

int Cabana::CommunicationPlan< MemorySpace >::neighborRank ( const int neighbor ) const

inline

Given a local neighbor id get its rank in the MPI communicator.

Parameters

neighbor The local id of the neighbor to get the rank for.

Returns: The MPI rank of the neighbor with the given local id.

◆ numExport()

template<class MemorySpace>

std::size_t Cabana::CommunicationPlan< MemorySpace >::numExport ( const int neighbor ) const

inline

Get the number of elements this rank will export to a given neighbor.

Parameters

neighbor The local id of the neighbor to get the number of exports for.

Returns: The number of elements this rank will export to the neighbor with the given local id.

◆ numImport()

template<class MemorySpace>

std::size_t Cabana::CommunicationPlan< MemorySpace >::numImport ( const int neighbor ) const

inline

Get the number of elements this rank will import from a given neighbor.

Parameters

neighbor The local id of the neighbor to get the number of imports for.

Returns: The number of elements this rank will import from the neighbor with the given local id.

◆ numNeighbor()

template<class MemorySpace>

int Cabana::CommunicationPlan< MemorySpace >::numNeighbor ( ) const

inline

Get the number of neighbor ranks that this rank will communicate with.

Returns: The number of MPI ranks that will exchange data with this rank.

◆ totalNumExport()

template<class MemorySpace>

std::size_t Cabana::CommunicationPlan< MemorySpace >::totalNumExport ( ) const

inline

Get the total number of exports this rank will do.

Returns: The total number of elements this rank will export to its neighbors.

◆ totalNumImport()

template<class MemorySpace>

std::size_t Cabana::CommunicationPlan< MemorySpace >::totalNumImport ( ) const

inline

Get the total number of imports this rank will do.

Returns: The total number of elements this rank will import from its neighhbors.

The documentation for this class was generated from the following file:

core/src/Cabana_CommunicationPlan.hpp

Public Types

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ CommunicationPlan()

Member Function Documentation

◆ createExportSteering() [1/2]

◆ createExportSteering() [2/2]

◆ createWithoutTopology() [1/4]

◆ createWithoutTopology() [2/4]

◆ createWithoutTopology() [3/4]

◆ createWithoutTopology() [4/4]

◆ createWithTopology() [1/4]

◆ createWithTopology() [2/4]

◆ createWithTopology() [3/4]

◆ createWithTopology() [4/4]

◆ exportSize()

◆ getExportSteering()

◆ neighborRank()

◆ numExport()

◆ numImport()

◆ numNeighbor()

◆ totalNumExport()

◆ totalNumImport()