132 MPI_Topo_test(this->globalComm, &status);
133 if (status != MPI_CART) {
135 __FILE__, __func__, __LINE__,
136 "Cartesian MPI communicator required, passed communicator not "
140 int dim = this->getDimension();
144 MPI_Cart_get(this->globalComm, dim, this->global_dims.data(),
145 this->periodicity.data(), this->local_coords.data());
148 MPI_Cart_rank(this->globalComm, this->local_coords.data(), &this->localRank);
151 for (
int i = 0; i < dim; ++i) {
152 MPI_Comm_split(this->globalComm, this->local_coords.at(i), 0,
157 if (std::is_same<T, double>::value)
158 MPIDataTypeT = MPI_DOUBLE;
159 else if (std::is_same<T, float>::value)
160 MPIDataTypeT = MPI_FLOAT;
161 else if (std::is_same<T, int>::value)
162 MPIDataTypeT = MPI_INT;
163 else if (std::is_same<T, long>::value)
164 MPIDataTypeT = MPI_LONG;
167 if (std::is_same<W, double>::value)
168 MPIDataTypeW = MPI_DOUBLE;
169 else if (std::is_same<W, float>::value)
170 MPIDataTypeW = MPI_FLOAT;
171 else if (std::is_same<W, int>::value)
172 MPIDataTypeW = MPI_INT;
173 else if (std::is_same<W, long>::value)
174 MPIDataTypeW = MPI_LONG;
177 int rank_left, rank_right;
180 for (
int i = 0; i < dim; ++i) {
181 MPI_Cart_shift(this->globalComm, i, 1, &rank_left, &rank_right);
182 neighbors.push_back(rank_left);
183 neighbors.push_back(rank_right);
184 nNeighbors[2 * i] = 1;
185 nNeighbors[2 * i + 1] = 1;
190 int dim = this->getDimension();
191 std::vector<Point<T>> newVertices = this->vertices;
192 this->prevVertices = this->vertices;
194 bool localIsSum = reductionMode == MPI_SUM;
195 bool localIsMax = reductionMode == MPI_MAX;
198 for (
int i = 0; i < dim; ++i) {
201 MPI_Allreduce(this->getWork().data(), &work_local_plane, 1, MPIDataTypeW,
202 reductionMode, communicators.at(i));
210 int rank_left, rank_right;
212 MPI_Cart_shift(this->globalComm, i, 1, &rank_left, &rank_right);
215 MPI_Request sreq, rreq;
216 MPI_Status sstat, rstat;
218 MPI_Irecv(&remote_work, 1, MPIDataTypeW, rank_right, 0, this->globalComm,
220 MPI_Isend(&work_local_plane, 1, MPIDataTypeW, rank_left, 0,
221 this->globalComm, &sreq);
222 MPI_Wait(&sreq, &sstat);
223 MPI_Wait(&rreq, &rstat);
227 local_size = this->prevVertices.at(1)[i] - this->prevVertices.at(0)[i];
229 MPI_Irecv(&remote_size, 1, MPIDataTypeT, rank_right, 0, this->globalComm,
231 MPI_Isend(&local_size, 1, MPIDataTypeT, rank_left, 0, this->globalComm,
233 MPI_Wait(&sreq, &sstat);
234 MPI_Wait(&rreq, &rstat);
239 std::max(4.1, 2.0 * (1.0 + std::max(local_size, remote_size) /
240 std::min(local_size, remote_size)));
243 rank_right, this->local_coords.at(i), this->global_dims.at(i),
244 work_local_plane, remote_work, local_size, remote_size, this->gamma,
248 T remote_shift = (T)0;
250 MPI_Irecv(&remote_shift, 1, MPIDataTypeT, rank_left, 0, this->globalComm,
252 MPI_Isend(&shift, 1, MPIDataTypeT, rank_right, 0, this->globalComm, &sreq);
253 MPI_Wait(&sreq, &sstat);
254 MPI_Wait(&rreq, &rstat);
257 if (rank_left != MPI_PROC_NULL && this->local_coords[i] != 0)
258 newVertices.at(0)[i] = this->prevVertices.at(0)[i] + remote_shift;
260 newVertices.at(0)[i] = this->prevVertices.at(0)[i];
263 if (rank_right != MPI_PROC_NULL &&
264 this->local_coords[i] != this->global_dims[i] - 1)
265 newVertices.at(1)[i] = this->prevVertices.at(1)[i] + shift;
267 newVertices.at(1)[i] = this->prevVertices.at(1)[i];
270 this->setVertices(newVertices);