DOLFIN-X
DOLFIN-X C++ interface
IndexMap.h
1 // Copyright (C) 2015-2019 Chris Richardson, Garth N. Wells and Igor Baratta
2 //
3 // This file is part of DOLFINX (https://www.fenicsproject.org)
4 //
5 // SPDX-License-Identifier: LGPL-3.0-or-later
6 
7 #pragma once
8 
9 #include <Eigen/Dense>
10 #include <array>
11 #include <cstdint>
12 #include <dolfinx/common/MPI.h>
13 #include <map>
14 #include <set>
15 #include <vector>
16 
17 namespace dolfinx::common
18 {
19 
26 
27 class IndexMap
28 {
29 public:
31  enum class Mode
32  {
33  insert,
34  add
35  };
36 
46  IndexMap(MPI_Comm mpi_comm, std::int32_t local_size,
47  const std::vector<std::int64_t>& ghosts, int block_size);
48 
58  IndexMap(
59  MPI_Comm mpi_comm, std::int32_t local_size,
60  const Eigen::Ref<const Eigen::Array<std::int64_t, Eigen::Dynamic, 1>>&
61  ghosts,
62  int block_size);
63 
65  IndexMap(const IndexMap& map) = delete;
66 
68  IndexMap(IndexMap&& map) = default;
69 
71  ~IndexMap() = default;
72 
74  std::array<std::int64_t, 2> local_range() const;
75 
77  int block_size() const;
78 
80  std::int32_t num_ghosts() const;
81 
83  std::int32_t size_local() const;
84 
86  std::int64_t size_global() const;
87 
90  const Eigen::Array<std::int64_t, Eigen::Dynamic, 1>& ghosts() const;
91 
98  Eigen::Array<std::int64_t, Eigen::Dynamic, 1> local_to_global(
99  const Eigen::Ref<const Eigen::Array<std::int32_t, Eigen::Dynamic, 1>>&
100  indices,
101  bool blocked = true) const;
102 
112  std::vector<std::int64_t>
113  local_to_global(const std::vector<std::int32_t>& indices,
114  bool blocked = true) const;
115 
122  std::vector<std::int32_t>
123  global_to_local(const std::vector<std::int64_t>& indices,
124  bool blocked = true) const;
125 
132  std::vector<std::int32_t> global_to_local(
133  const Eigen::Ref<const Eigen::Array<std::int64_t, Eigen::Dynamic, 1>>&
134  indices,
135  bool blocked = true) const;
136 
140  std::vector<std::int64_t> global_indices(bool blocked = true) const;
141 
144  std::int64_t local_to_global(std::int32_t local_index) const
145  {
146  assert(local_index >= 0);
147  const std::int32_t local_size
148  = _all_ranges[_myrank + 1] - _all_ranges[_myrank];
149  if (local_index < local_size)
150  {
151  const std::int64_t global_offset = _all_ranges[_myrank];
152  return global_offset + local_index;
153  }
154  else
155  {
156  assert((local_index - local_size) < _ghosts.size());
157  return _ghosts[local_index - local_size];
158  }
159  }
160 
165  const std::vector<std::int32_t>& forward_indices() const
166  {
167  return _forward_indices;
168  }
169 
171  Eigen::Array<std::int32_t, Eigen::Dynamic, 1> ghost_owners() const;
172 
174  int owner(std::int64_t global_index) const;
175 
178  Eigen::Array<std::int64_t, Eigen::Dynamic, 1>
179  indices(bool unroll_block) const;
180 
183  MPI_Comm mpi_comm() const;
184 
186  const std::vector<std::int32_t>& neighbours() const;
187 
193  std::map<std::int32_t, std::set<std::int32_t>> compute_shared_indices() const;
194 
204  void scatter_fwd(const std::vector<std::int64_t>& local_data,
205  std::vector<std::int64_t>& remote_data, int n) const;
206 
216  void scatter_fwd(const std::vector<std::int32_t>& local_data,
217  std::vector<std::int32_t>& remote_data, int n) const;
218 
228  std::vector<std::int64_t>
229  scatter_fwd(const std::vector<std::int64_t>& local_data, int n) const;
230 
239  std::vector<std::int32_t>
240  scatter_fwd(const std::vector<std::int32_t>& local_data, int n) const;
241 
250  void scatter_rev(std::vector<std::int64_t>& local_data,
251  const std::vector<std::int64_t>& remote_data, int n,
252  IndexMap::Mode op) const;
253 
262  void scatter_rev(std::vector<std::int32_t>& local_data,
263  const std::vector<std::int32_t>& remote_data, int n,
264  IndexMap::Mode op) const;
265 
266 private:
267  int _block_size;
268 
269  // MPI Communicator
270  dolfinx::MPI::Comm _mpi_comm;
271 
272  // MPI Communicator for neighbourhood only
273  // mutable MPI_Comm _neighbour_comm;
274 
275  // Store neighbours so neighbourhood communicator can be re-built
276  std::vector<std::int32_t> _neighbours;
277 
278  // Cache rank on mpi_comm (otherwise calls to MPI_Comm_rank can be
279  // excessive)
280  int _myrank;
281 
282 public:
283  // FIXME: This could get big for large process counts
284  // Range of ownership of index for all processes
285  std::vector<std::int64_t> _all_ranges;
286 
287 private:
288  // Local-to-global map for ghost indices
289  Eigen::Array<std::int64_t, Eigen::Dynamic, 1> _ghosts;
290 
291  // Owning neighbour for each ghost index
292  Eigen::Array<std::int32_t, Eigen::Dynamic, 1> _ghost_owners;
293 
294  // Number of indices to send to each neighbour process (ghost ->
295  // owner, i.e. forward mode scatter)
296  std::vector<std::int32_t> _forward_sizes;
297 
298  // "Owned" local indices shared with neighbour processes
299  std::vector<std::int32_t> _forward_indices;
300 
301  template <typename T>
302  void scatter_fwd_impl(const std::vector<T>& local_data,
303  std::vector<T>& remote_data, int n) const;
304  template <typename T>
305  void scatter_rev_impl(std::vector<T>& local_data,
306  const std::vector<T>& remote_data, int n,
307  Mode op) const;
308 };
309 
310 } // namespace dolfinx::common
dolfinx::common::IndexMap::indices
Eigen::Array< std::int64_t, Eigen::Dynamic, 1 > indices(bool unroll_block) const
Return array of global indices for all indices on this process, including ghosts.
Definition: IndexMap.cpp:346
dolfinx::common::IndexMap::~IndexMap
~IndexMap()=default
Destructor.
dolfinx::common::IndexMap
This class represents the distribution index arrays across processes. An index array is a contiguous ...
Definition: IndexMap.h:27
dolfinx::common::IndexMap::local_range
std::array< std::int64_t, 2 > local_range() const
Range of indices (global) owned by this process.
Definition: IndexMap.cpp:191
dolfinx::common::IndexMap::compute_shared_indices
std::map< std::int32_t, std::set< std::int32_t > > compute_shared_indices() const
Definition: IndexMap.cpp:371
dolfinx::common::IndexMap::ghosts
const Eigen::Array< std::int64_t, Eigen::Dynamic, 1 > & ghosts() const
Local-to-global map for ghosts (local indexing beyond end of local range)
Definition: IndexMap.cpp:207
dolfinx::common::IndexMap::forward_indices
const std::vector< std::int32_t > & forward_indices() const
Definition: IndexMap.h:165
dolfinx::common::IndexMap::ghost_owners
Eigen::Array< std::int32_t, Eigen::Dynamic, 1 > ghost_owners() const
Owner rank (on global communicator) of each ghost entry.
Definition: IndexMap.cpp:317
dolfinx::common::IndexMap::size_global
std::int64_t size_global() const
Number indices across communicator.
Definition: IndexMap.cpp:205
dolfinx::common::IndexMap::local_to_global
Eigen::Array< std::int64_t, Eigen::Dynamic, 1 > local_to_global(const Eigen::Ref< const Eigen::Array< std::int32_t, Eigen::Dynamic, 1 >> &indices, bool blocked=true) const
Compute global indices for array of local indices.
Definition: IndexMap.cpp:212
dolfinx::common::IndexMap::size_local
std::int32_t size_local() const
Number of indices owned by on this process.
Definition: IndexMap.cpp:200
dolfinx::common::IndexMap::scatter_fwd
void scatter_fwd(const std::vector< std::int64_t > &local_data, std::vector< std::int64_t > &remote_data, int n) const
Send n values for each index that is owned to processes that have the index as a ghost....
Definition: IndexMap.cpp:457
dolfinx::common::IndexMap::global_to_local
std::vector< std::int32_t > global_to_local(const std::vector< std::int64_t > &indices, bool blocked=true) const
Compute local indices for array of global indices.
Definition: IndexMap.cpp:266
dolfinx::common::IndexMap::mpi_comm
MPI_Comm mpi_comm() const
Return MPI communicator.
Definition: IndexMap.cpp:364
dolfinx::common::IndexMap::scatter_rev
void scatter_rev(std::vector< std::int64_t > &local_data, const std::vector< std::int64_t > &remote_data, int n, IndexMap::Mode op) const
Send n values for each ghost index to owning to the process.
Definition: IndexMap.cpp:485
dolfinx::common
Miscellaneous classes, functions and types.
dolfinx::MPI::Comm
A duplicate MPI communicator and manage lifetime of the communicator.
Definition: MPI.h:34
dolfinx::common::IndexMap::num_ghosts
std::int32_t num_ghosts() const
Number of ghost indices on this process.
Definition: IndexMap.cpp:198
dolfinx::common::IndexMap::IndexMap
IndexMap(MPI_Comm mpi_comm, std::int32_t local_size, const std::vector< std::int64_t > &ghosts, int block_size)
Create Index map with local_size owned blocks on this process, and blocks have size block_size.
Definition: IndexMap.cpp:58
dolfinx::common::IndexMap::neighbours
const std::vector< std::int32_t > & neighbours() const
Neighbors for neigborhood communicator.
Definition: IndexMap.cpp:366
dolfinx::common::IndexMap::Mode
Mode
Mode for reverse scatter operation.
Definition: IndexMap.h:31
dolfinx::common::IndexMap::block_size
int block_size() const
Block size.
Definition: IndexMap.cpp:196
dolfinx::common::IndexMap::local_to_global
std::int64_t local_to_global(std::int32_t local_index) const
Definition: IndexMap.h:144
dolfinx::common::IndexMap::global_indices
std::vector< std::int64_t > global_indices(bool blocked=true) const
Global indices.
Definition: IndexMap.cpp:245
dolfinx::common::IndexMap::owner
int owner(std::int64_t global_index) const
Get process that owns index (global block index)
Definition: IndexMap.cpp:310