10 #include <dolfinx/common/IndexMap.h>
11 #include <dolfinx/common/Timer.h>
12 #include <dolfinx/graph/AdjacencyList.h>
49 static std::tuple<std::vector<std::int32_t>, std::vector<std::int64_t>,
52 const std::vector<std::int64_t>& global_indices,
53 const std::vector<bool>& shared_indices);
63 static std::pair<graph::AdjacencyList<std::int32_t>,
64 std::vector<std::int64_t>>
80 const std::vector<std::int64_t>& local_to_global_links,
81 const std::vector<bool>& shared_links);
94 static std::tuple<graph::AdjacencyList<std::int64_t>, std::vector<int>,
95 std::vector<std::int64_t>, std::vector<int>>
106 static std::vector<std::int64_t>
108 const std::vector<std::int64_t>& global_indices,
109 const std::vector<int>& ghost_owners);
120 template <
typename T>
121 static Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
122 distribute_data(MPI_Comm comm,
const std::vector<std::int64_t>& indices,
123 const Eigen::Ref<
const Eigen::Array<
124 T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>& x);
149 static std::vector<std::int32_t>
151 const std::vector<std::int64_t>& local1_to_global);
157 template <
typename T>
158 Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
160 MPI_Comm comm,
const std::vector<std::int64_t>& indices,
161 const Eigen::Ref<
const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic,
162 Eigen::RowMajor>>& x)
164 common::Timer timer(
"Fetch float data from remote processes");
166 const std::int64_t num_points_local = x.rows();
169 std::vector<std::int64_t> global_sizes(size);
170 MPI_Allgather(&num_points_local, 1, MPI_INT64_T, global_sizes.data(), 1,
172 std::vector<std::int64_t> global_offsets(size + 1, 0);
173 std::partial_sum(global_sizes.begin(), global_sizes.end(),
174 global_offsets.begin() + 1);
177 std::vector<int> number_index_send(size, 0);
178 std::vector<int> index_owner(indices.size());
179 std::vector<int> index_order(indices.size());
180 std::iota(index_order.begin(), index_order.end(), 0);
181 std::sort(index_order.begin(), index_order.end(),
182 [&indices](
int a,
int b) { return (indices[a] < indices[b]); });
185 for (std::size_t i = 0; i < index_order.size(); ++i)
187 int j = index_order[i];
188 while (indices[j] >= global_offsets[p + 1])
191 number_index_send[p]++;
195 std::vector<int> disp_index_send(size + 1, 0);
196 std::partial_sum(number_index_send.begin(), number_index_send.end(),
197 disp_index_send.begin() + 1);
200 std::vector<std::int64_t> indices_send(disp_index_send.back());
201 std::vector<int> disp_tmp = disp_index_send;
202 for (std::size_t i = 0; i < indices.size(); ++i)
204 const int owner = index_owner[i];
205 indices_send[disp_tmp[owner]++] = indices[i];
209 std::vector<int> number_index_recv(size);
210 MPI_Alltoall(number_index_send.data(), 1, MPI_INT, number_index_recv.data(),
214 std::vector<int> disp_index_recv(size + 1, 0);
215 std::partial_sum(number_index_recv.begin(), number_index_recv.end(),
216 disp_index_recv.begin() + 1);
219 std::vector<std::int64_t> indices_recv(disp_index_recv.back());
220 MPI_Alltoallv(indices_send.data(), number_index_send.data(),
221 disp_index_send.data(), MPI_INT64_T, indices_recv.data(),
222 number_index_recv.data(), disp_index_recv.data(), MPI_INT64_T,
225 const int item_size = x.cols();
226 assert(item_size != 0);
228 Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> x_return(
229 indices_recv.size(), item_size);
230 for (
int p = 0; p < size; ++p)
232 for (
int i = disp_index_recv[p]; i < disp_index_recv[p + 1]; ++i)
234 const std::int32_t index_local = indices_recv[i] - global_offsets[rank];
235 assert(index_local >= 0);
236 x_return.row(i) = x.row(index_local);
240 MPI_Datatype compound_type;
241 MPI_Type_contiguous(item_size, dolfinx::MPI::mpi_type<T>(), &compound_type);
242 MPI_Type_commit(&compound_type);
245 Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> my_x(
246 disp_index_send.back(), item_size);
247 MPI_Alltoallv(x_return.data(), number_index_recv.data(),
248 disp_index_recv.data(), compound_type, my_x.data(),
249 number_index_send.data(), disp_index_send.data(), compound_type,