Sierra Toolkit  Version of the Day
UseCase_Rebal_1.cpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 #include <use_cases/UseCase_Rebal_1.hpp>
10 
11 #include <stk_util/parallel/Parallel.hpp>
12 #include <stk_util/parallel/ParallelReduce.hpp>
13 
14 #include <stk_mesh/base/FieldData.hpp>
15 #include <stk_mesh/base/GetEntities.hpp>
16 
20 
21 #include <stk_rebalance_utils/RebalanceUtils.hpp>
22 
23 //----------------------------------------------------------------------
24 
25 using namespace stk_classic::mesh::fixtures;
26 
28 
29 namespace stk_classic {
30 namespace rebalance {
31 namespace use_cases {
32 
33 bool test_unequal_weights( stk_classic::ParallelMachine pm )
34 {
35  const unsigned p_size = stk_classic::parallel_machine_size(pm);
36  const unsigned p_rank = stk_classic::parallel_machine_rank(pm);
37 
38  const unsigned ngx = p_size*(p_size+1)/2;
39 
40  unsigned nx = 0;
41  if( 0 == p_rank )
42  nx = ngx;
43  unsigned ny = 1;
44  unsigned nz = 1;
45 
46  stk_classic::mesh::fixtures::HexFixture fixture(pm, nx, ny, nz);
47 
48  stk_classic::mesh::fem::FEMMetaData & fem_meta = fixture.m_fem_meta;
49  stk_classic::mesh::BulkData & bulk = fixture.m_bulk_data;
50 
51  // Put weights field on all elements
52  const stk_classic::mesh::EntityRank element_rank = fem_meta.element_rank();
53  ScalarField & weight_field( fem_meta.declare_field< ScalarField >( "element_weights" ) );
54  stk_classic::mesh::put_field(weight_field , element_rank , fem_meta.universal_part() );
55 
56  fem_meta.commit();
57 
58  bulk.modification_begin();
59 
60  // Initially put all elements on proc 0
61  std::vector<stk_classic::mesh::EntityId> my_element_ids;
62  for ( unsigned i = 0 ; i < nx*ny*nz; ++i )
63  my_element_ids.push_back(i+1);
64 
65  fixture.generate_mesh(my_element_ids);
66 
67  // Assign weights so that a perfect rebalance is possible so long as the rebalancer can figure out
68  // to put p_rank+1 elements on proc = p_rank based on these weights.
69  unsigned nslabs = 0;
70  if( 0 == p_rank ) {
71  for ( unsigned l = 1 ; l <= p_size ; ++l ) {
72  for ( unsigned k = 0 ; k < nz ; ++k ) {
73  for ( unsigned j = 0 ; j < ny ; ++j ) {
74  for ( unsigned i = 0 ; i < l ; ++i ) {
75  const stk_classic::mesh::EntityId elem_id = 1 + nslabs + i + j*ngx + k*ngx*ny;
76  stk_classic::mesh::Entity * elem = bulk.get_entity(element_rank, elem_id);
77  double * const e_weight = stk_classic::mesh::field_data( weight_field , *elem );
78  *e_weight = double(ngx) / double(l);
79  }
80  }
81  }
82  nslabs += l;
83  }
84  }
85  // end assign weights
86 
87  bulk.modification_end();
88 
89  // Use Zoltan to determine new partition
90  Teuchos::ParameterList emptyList;
91  stk_classic::rebalance::Zoltan zoltan_partition(pm, fixture.m_spatial_dimension, emptyList);
92 
93  stk_classic::mesh::Selector selector(fem_meta.universal_part());
94 
95  stk_classic::rebalance::rebalance(bulk, selector, &fixture.m_coord_field, &weight_field, zoltan_partition);
96 
97  const double imbalance_threshold = stk_classic::rebalance::check_balance(bulk, &weight_field, element_rank);
98  const bool do_rebal = 1.5 < imbalance_threshold;
99 
100  if( 0 == p_rank )
101  std::cerr << std::endl
102  << "imbalance_threshold after rebalance = " << imbalance_threshold << ", " << do_rebal << std::endl;
103 
104  stk_classic::mesh::Selector owned_selector = fem_meta.locally_owned_part();
105  size_t num_local_elems = stk_classic::mesh::count_selected_entities(owned_selector, bulk.buckets(element_rank));
106 
107  // Check that we satisfy our threshhold
108  bool result = true;
109  if( 4 > p_size )
110  {
111  result = (fabs(imbalance_threshold - 1.0) < 1.e-8);
112  result = result & (num_local_elems == p_rank+1);
113  }
114  else
115  {
116  // Would like to put something here, but Zoltan using its default algorithm (RCB)
117  // isn't able to do an adequate job rebalancing
118  result = !do_rebal;
119  }
120 
121  return result;
122 }
123 
124 } //namespace use_cases
125 } //namespace rebalance
126 } //namespace stk_classic
127 
128 
Part & locally_owned_part() const
Subset for the problem domain that is owned by the local process. Ghost entities are not members of t...
FEMMetaData is a class that implements a Finite Element Method skin on top of the Sierra Tool Kit Met...
Definition: FEMMetaData.hpp:54
bool rebalance(mesh::BulkData &bulk_data, const mesh::Selector &selector, const VectorField *coord_ref, const ScalarField *elem_weight_ref, Partition &partition, const stk_classic::mesh::EntityRank rank=stk_classic::mesh::InvalidEntityRank)
Rebalance with a Partition object.
Definition: Rebalance.cpp:164
unsigned count_selected_entities(const Selector &selector, const std::vector< Bucket * > &input_buckets)
Count entities in selected buckets (selected by the given selector instance), and sorted by ID...
Definition: GetEntities.cpp:59
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
Definition: FieldData.hpp:116
EntityRank element_rank() const
Returns the element rank which is always equal to spatial dimension.
Part & universal_part() const
Universal subset for the problem domain. All other parts are a subset of the universal part...
This is a class for selecting buckets based on a set of meshparts and set logic.
Definition: Selector.hpp:112
const std::vector< Bucket * > & buckets(EntityRank rank) const
Query all buckets of a given entity rank.
Definition: BulkData.hpp:195
field_type & put_field(field_type &field, EntityRank entity_rank, const Part &part, const void *init_value=NULL)
Declare a field to exist for a given entity type and Part.
Entity * get_entity(EntityRank entity_rank, EntityId entity_id) const
Get entity with a given key.
Definition: BulkData.hpp:211
unsigned parallel_machine_rank(ParallelMachine parallel_machine)
Member function parallel_machine_rank ...
Definition: Parallel.cpp:29
bool modification_end()
Parallel synchronization of modifications and transition to the guaranteed parallel consistent state...
bool modification_begin()
Begin a modification phase during which the mesh bulk data could become parallel inconsistent. This is a parallel synchronous call. The first time this method is called the mesh meta data is verified to be committed and parallel consistent. An exception is thrown if this verification fails.
Definition: BulkData.cpp:172
unsigned parallel_machine_size(ParallelMachine parallel_machine)
Member function parallel_machine_size ...
Definition: Parallel.cpp:18
Manager for an integrated collection of entities, entity relations, and buckets of field data...
Definition: BulkData.hpp:49
void commit()
Commit the part and field declarations so that the meta data manager can be used to create mesh bulk ...
field_type & declare_field(const std::string &name, unsigned number_of_states=1)
Declare a field of the given field_type, test name, and number of states.
A fundamental unit within the discretization of a problem domain, including but not limited to nodes...
Definition: Entity.hpp:120
Sierra Toolkit.
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
For partitioning of mesh entities over a processing grid.
Static functions for dynamic load balancing.