MueLu  Version of the Day
MueLu_Zoltan2Interface_def.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // MueLu: A package for multigrid based preconditioning
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact
39 // Jonathan Hu (jhu@sandia.gov)
40 // Andrey Prokopenko (aprokop@sandia.gov)
41 // Ray Tuminaro (rstumin@sandia.gov)
42 //
43 // ***********************************************************************
44 //
45 // @HEADER
46 #ifndef MUELU_ZOLTAN2INTERFACE_DEF_HPP
47 #define MUELU_ZOLTAN2INTERFACE_DEF_HPP
48 
49 #include <sstream>
50 #include <set>
51 
53 #if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI)
54 
55 #include <Zoltan2_XpetraMultiVectorAdapter.hpp>
56 #include <Zoltan2_XpetraCrsGraphAdapter.hpp>
57 #include <Zoltan2_PartitioningProblem.hpp>
58 
59 #include <Teuchos_Utils.hpp>
61 #include <Teuchos_OpaqueWrapper.hpp>
62 
63 #include "MueLu_Level.hpp"
64 #include "MueLu_Exceptions.hpp"
65 #include "MueLu_Monitor.hpp"
66 #include "MueLu_Utilities.hpp"
67 
68 namespace MueLu {
69 
70  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
72  defaultZoltan2Params = rcp(new ParameterList());
73  defaultZoltan2Params->set("algorithm", "multijagged");
74  defaultZoltan2Params->set("partitioning_approach", "partition");
75 
76  // Improve scaling for communication bound algorithms by premigrating
77  // coordinates to a subset of processors.
78  // For more information, see Github issue #1538
79  defaultZoltan2Params->set("mj_premigration_option", 1);
80  }
81 
82  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
84  RCP<ParameterList> validParamList = rcp(new ParameterList());
85 
86  validParamList->set< RCP<const FactoryBase> > ("A", Teuchos::null, "Factory of the matrix A");
87  validParamList->set< RCP<const FactoryBase> > ("number of partitions", Teuchos::null, "Instance of RepartitionHeuristicFactory.");
88  validParamList->set< RCP<const FactoryBase> > ("Coordinates", Teuchos::null, "Factory of the coordinates");
89  validParamList->set< RCP<const ParameterList> > ("ParameterList", Teuchos::null, "Zoltan2 parameters");
90 
91  return validParamList;
92  }
93 
94 
95  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
97  Input(currentLevel, "A");
98  Input(currentLevel, "number of partitions");
99  const ParameterList& pL = GetParameterList();
100  // We do this dance, because we don't want "ParameterList" to be marked as used.
101  // Is there a better way?
102  Teuchos::ParameterEntry entry = pL.getEntry("ParameterList");
103  RCP<const Teuchos::ParameterList> providedList = Teuchos::any_cast<RCP<const Teuchos::ParameterList> >(entry.getAny(false));
104  if (providedList != Teuchos::null && providedList->isType<std::string>("algorithm")) {
105  const std::string algo = providedList->get<std::string>("algorithm");
106  if (algo == "multijagged" || algo == "rcb")
107  Input(currentLevel, "Coordinates");
108  } else
109  Input(currentLevel, "Coordinates");
110  }
111 
112  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
114  FactoryMonitor m(*this, "Build", level);
115 
116  typedef typename Teuchos::ScalarTraits<SC>::magnitudeType real_type;
117  typedef typename Xpetra::MultiVector<real_type,LO,GO,NO> RealValuedMultiVector;
118 
119  RCP<Matrix> A = Get<RCP<Matrix> >(level, "A");
120  RCP<const Map> rowMap = A->getRowMap();
121  LO blkSize = A->GetFixedBlockSize();
122 
123  int numParts = Get<int>(level, "number of partitions");
124  if (numParts == 1 || numParts == -1) {
125  // Single processor, decomposition is trivial: all zeros
127  Set(level, "Partition", decomposition);
128  return;
129  }/* else if (numParts == -1) {
130  // No repartitioning
131  RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Teuchos::null; //Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
132  //decomposition->putScalar(Teuchos::as<Scalar>(rowMap->getComm()->getRank()));
133  Set(level, "Partition", decomposition);
134  return;
135  }*/
136 
137  const ParameterList& pL = GetParameterList();
138 
139  RCP<const ParameterList> providedList = pL.get<RCP<const ParameterList> >("ParameterList");
140  ParameterList Zoltan2Params;
141  if (providedList != Teuchos::null)
142  Zoltan2Params = *providedList;
143 
144  // Merge defalt Zoltan2 parameters with user provided
145  // If default and user parameters contain the same parameter name, user one is always preferred
146  for (ParameterList::ConstIterator param = defaultZoltan2Params->begin(); param != defaultZoltan2Params->end(); param++) {
147  const std::string& pName = defaultZoltan2Params->name(param);
148  if (!Zoltan2Params.isParameter(pName))
149  Zoltan2Params.setEntry(pName, defaultZoltan2Params->getEntry(pName));
150  }
151  Zoltan2Params.set("num_global_parts", Teuchos::as<int>(numParts));
152 
153  GetOStream(Runtime0) << "Zoltan2 parameters:\n----------\n" << Zoltan2Params << "----------" << std::endl;
154 
155  const std::string& algo = Zoltan2Params.get<std::string>("algorithm");
156 
157  if (algo == "multijagged" || algo == "rcb") {
158 
159  RCP<RealValuedMultiVector> coords = Get<RCP<RealValuedMultiVector> >(level, "Coordinates");
160  RCP<const Map> map = coords->getMap();
161  GO numElements = map->getNodeNumElements();
162 
163  // Check that the number of local coordinates is consistent with the #rows in A
164  TEUCHOS_TEST_FOR_EXCEPTION(rowMap->getNodeNumElements()/blkSize != coords->getLocalLength(), Exceptions::Incompatible,
165  "Coordinate vector length (" + toString(coords->getLocalLength()) << " is incompatible with number of block rows in A ("
166  + toString(rowMap->getNodeNumElements()/blkSize) + "The vector length should be the same as the number of mesh points.");
167 #ifdef HAVE_MUELU_DEBUG
168  GO indexBase = rowMap->getIndexBase();
169  GetOStream(Runtime0) << "Checking consistence of row and coordinates maps" << std::endl;
170  // Make sure that logical blocks in row map coincide with logical nodes in coordinates map
171  ArrayView<const GO> rowElements = rowMap->getNodeElementList();
172  ArrayView<const GO> coordsElements = map ->getNodeElementList();
173  for (LO i = 0; i < Teuchos::as<LO>(numElements); i++)
174  TEUCHOS_TEST_FOR_EXCEPTION((coordsElements[i]-indexBase)*blkSize + indexBase != rowElements[i*blkSize],
175  Exceptions::RuntimeError, "i = " << i << ", coords GID = " << coordsElements[i]
176  << ", row GID = " << rowElements[i*blkSize] << ", blkSize = " << blkSize << std::endl);
177 #endif
178 
179  typedef Zoltan2::XpetraMultiVectorAdapter<RealValuedMultiVector> InputAdapterType;
180  typedef Zoltan2::PartitioningProblem<InputAdapterType> ProblemType;
181 
182  Array<double> weightsPerRow(numElements);
183  for (LO i = 0; i < numElements; i++) {
184  weightsPerRow[i] = 0.0;
185 
186  for (LO j = 0; j < blkSize; j++) {
187  weightsPerRow[i] += A->getNumEntriesInLocalRow(i*blkSize+j);
188  }
189  }
190 
191  std::vector<int> strides;
192  std::vector<const double*> weights(1, weightsPerRow.getRawPtr());
193 
194  RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
195  RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
196 
197  InputAdapterType adapter(coords, weights, strides);
198  RCP<ProblemType> problem(new ProblemType(&adapter, &Zoltan2Params, (*zoltanComm)()));
199 
200  {
201  SubFactoryMonitor m1(*this, "Zoltan2 " + toString(algo), level);
202  problem->solve();
203  }
204 
206  ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
207 
208  const typename InputAdapterType::part_t * parts = problem->getSolution().getPartListView();
209 
210  for (GO i = 0; i < numElements; i++) {
211  int partNum = parts[i];
212 
213  for (LO j = 0; j < blkSize; j++)
214  decompEntries[i*blkSize + j] = partNum;
215  }
216 
217  Set(level, "Partition", decomposition);
218 
219  } else {
220 
221  GO numElements = rowMap->getNodeNumElements();
222 
223  typedef Zoltan2::XpetraCrsGraphAdapter<CrsGraph> InputAdapterType;
224  typedef Zoltan2::PartitioningProblem<InputAdapterType> ProblemType;
225 
226  RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
227  RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
228 
229  InputAdapterType adapter(A->getCrsGraph());
230  RCP<ProblemType> problem(new ProblemType(&adapter, &Zoltan2Params, (*zoltanComm)()));
231 
232  {
233  SubFactoryMonitor m1(*this, "Zoltan2 " + toString(algo), level);
234  problem->solve();
235  }
236 
238  ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
239 
240  const typename InputAdapterType::part_t * parts = problem->getSolution().getPartListView();
241 
242  for (GO i = 0; i < numElements; i++) {
243  int partNum = parts[i];
244 
245  for (LO j = 0; j < blkSize; j++)
246  decompEntries[i*blkSize + j] = partNum;
247  }
248 
249  Set(level, "Partition", decomposition);
250  }
251  }
252 
253 } //namespace MueLu
254 
255 #endif //if defined(HAVE_MUELU_ZOLTAN2) && defined(HAVE_MPI)
256 
257 #endif // MUELU_ZOLTAN2INTERFACE_DEF_HPP
void DeclareInput(Level &currentLevel) const
Specifies the data that this class needs, and the factories that generate that data.
std::string toString(const T &what)
Little helper function to convert non-string types to strings.
GlobalOrdinal GO
T & get(const std::string &name, T def_value)
ParameterList & set(std::string const &name, T const &value, std::string const &docString="", RCP< const ParameterEntryValidator > const &validator=null)
Timer to be used in factories. Similar to Monitor but with additional timers.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
LocalOrdinal LO
One-liner description of what is happening.
double * getRawPtr()
Namespace for MueLu classes and methods.
Exception throws to report incompatible objects (like maps).
void Build(Level &currentLevel) const
Build an object with this factory.
T * get() const
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Class that holds all level-specific information.
Definition: MueLu_Level.hpp:99
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
params_t::ConstIterator ConstIterator
bool isType(const std::string &name) const
static RCP< Vector > Build(const Teuchos::RCP< const Map > &map, bool zeroOut=true)
const std::string & name() const
any & getAny(bool activeQry=true)
Exception throws to report errors in the internal logical of the program.
ParameterEntry & getEntry(const std::string &name)
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.