373 const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner,
375 :
comm(partitioner->get_mpi_communicator())
376 , comm_sm(communicator_sm)
377 , n_local_elements(partitioner->locally_owned_range().n_elements())
378 , n_ghost_elements(partitioner->ghost_indices().n_elements())
379 , n_global_elements(partitioner->locally_owned_range().size())
381#ifndef DEAL_II_WITH_MPI
387 const auto &is_locally_owned = partitioner->locally_owned_range();
388 const auto &is_locally_ghost = partitioner->ghost_indices();
389 const auto &ghost_indices_within_larger_ghost_set =
390 partitioner->ghost_indices_within_larger_ghost_set();
404 std::vector<unsigned int> ghost_indices_subset_data_ptr = {0};
405 std::vector<unsigned int> ghost_indices_subset_data_indices;
408 std::vector<unsigned int> import_indices_data_ptr = {0};
409 std::vector<unsigned int> import_indices_data_indices;
412 std::vector<unsigned int> sm_export_data_ptr = {0};
413 std::vector<unsigned int> sm_export_data_indices;
416 std::vector<unsigned int> sm_export_data_this_ptr = {0};
417 std::vector<unsigned int> sm_export_data_this_indices;
420 std::vector<unsigned int> sm_import_data_ptr = {};
421 std::vector<unsigned int> sm_import_data_indices;
424 std::vector<unsigned int> sm_import_data_this_ptr = {0};
425 std::vector<unsigned int> sm_import_data_this_indices;
428 const auto sm_ranks =
432 std::vector<unsigned int> owning_ranks_of_ghosts(
433 is_locally_ghost.n_elements());
436 process(is_locally_owned,
439 owning_ranks_of_ghosts,
444 std::pair<types::global_dof_index, types::global_dof_index>>,
445 std::vector<unsigned int>>
447 consensus_algorithm.
run(process,
comm);
451 std::vector<unsigned int> shifts_indices;
452 for (
const auto &pair : ghost_indices_within_larger_ghost_set)
453 for (
unsigned int k = pair.first; k < pair.second; ++k)
454 shifts_indices.push_back(k);
459 std::map<unsigned int, std::vector<types::global_dof_index>>
460 rank_to_local_indices;
462 for (
unsigned int i = 0; i < owning_ranks_of_ghosts.size(); ++i)
463 rank_to_local_indices[owning_ranks_of_ghosts[i]].push_back(i);
465 unsigned int compressed_offset = 0;
467 for (
const auto &rank_and_local_indices : rank_to_local_indices)
469 const auto sm_ranks_ptr = std::find(sm_ranks.begin(),
471 rank_and_local_indices.first);
473 if (sm_ranks_ptr == sm_ranks.end())
476 rank_and_local_indices.first,
477 shifts_indices[compressed_offset],
478 static_cast<unsigned int>(
479 rank_and_local_indices.second.size())
482 for (
unsigned int i = 0;
483 i < rank_and_local_indices.second.size();
485 ghost_indices_subset_data_indices.push_back(
486 shifts_indices[i + compressed_offset]);
488 ghost_indices_subset_data_ptr.push_back(
489 ghost_indices_subset_data_indices.size());
505 std::distance(sm_ranks.begin(), sm_ranks_ptr));
507 sm_export_data_ptr.push_back(
508 sm_export_data_ptr.back() +
509 rank_and_local_indices.second.size());
511 for (
unsigned int i = compressed_offset;
513 rank_and_local_indices.second.size() + compressed_offset;
515 sm_export_data_this_indices.push_back(
516 shifts_indices[i] + is_locally_owned.n_elements());
518 sm_export_data_this_ptr.push_back(
519 sm_export_data_this_indices.size());
521 compressed_offset += rank_and_local_indices.second.size();
524 sm_export_data_indices.resize(sm_export_data_ptr.back());
531 for (
const auto &rank_and_global_indices : rank_to_global_indices)
533 const auto sm_ranks_ptr =
534 std::find(sm_ranks.begin(),
536 rank_and_global_indices.first);
538 if (sm_ranks_ptr == sm_ranks.end())
541 rank_and_global_indices.first,
542 static_cast<unsigned int>(
543 import_indices_data_indices.size()),
544 static_cast<unsigned int>(
545 rank_and_global_indices.second.n_elements())
548 for (
const auto i : rank_and_global_indices.second)
549 import_indices_data_indices.push_back(
550 is_locally_owned.index_within_set(i));
552 import_indices_data_ptr.push_back(
553 import_indices_data_indices.size());
558 std::distance(sm_ranks.begin(), sm_ranks_ptr));
560 for (
const auto i : rank_and_global_indices.second)
561 sm_import_data_this_indices.push_back(
562 is_locally_owned.index_within_set(i));
564 sm_import_data_this_ptr.push_back(
565 sm_import_data_this_indices.size());
569 sm_import_data_ptr = sm_import_data_this_ptr;
570 sm_import_data_indices.resize(sm_import_data_this_ptr.back());
580 const int ierr = MPI_Isend(sm_export_data_this_indices.data() +
581 sm_export_data_this_ptr[i],
582 sm_export_data_this_ptr[i + 1] -
583 sm_export_data_this_ptr[i],
588 requests.data() + i);
595 MPI_Irecv(sm_import_data_indices.data() + sm_import_data_ptr[i],
596 sm_import_data_ptr[i + 1] - sm_import_data_ptr[i],
606 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
617 const int ierr = MPI_Isend(sm_import_data_this_indices.data() +
618 sm_import_data_this_ptr[i],
619 sm_import_data_this_ptr[i + 1] -
620 sm_import_data_this_ptr[i],
625 requests.data() + i);
632 MPI_Irecv(sm_export_data_indices.data() + sm_export_data_ptr[i],
633 sm_export_data_ptr[i + 1] - sm_export_data_ptr[i],
643 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
648 this->n_ghost_indices_in_larger_set_by_remote_rank =
653 ghost_indices_subset_data_ptr, ghost_indices_subset_data_indices);
661 import_indices_data_indices);
667 sm_export_data_indices);
671 sm_export_data_this_indices);
677 sm_import_data_indices);
681 sm_import_data_this_indices);