Sierra Toolkit  Version of the Day
BulkDataParallelVerify.cpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 
14 #include <set>
15 #include <stdexcept>
16 #include <iostream>
17 #include <sstream>
18 #include <algorithm>
19 
20 #include <stk_util/parallel/ParallelComm.hpp>
21 #include <stk_util/parallel/ParallelReduce.hpp>
22 
23 #include <stk_mesh/base/Ghosting.hpp>
24 #include <stk_mesh/base/BulkData.hpp>
25 #include <stk_mesh/base/MetaData.hpp>
26 #include <stk_mesh/base/FieldData.hpp>
27 #include <stk_mesh/base/EntityComm.hpp>
28 #include <stk_mesh/base/Comm.hpp>
29 
30 namespace stk_classic {
31 namespace mesh {
32 
33 namespace {
34 
35 bool verify_parallel_attributes( BulkData & M , std::ostream & error_log );
36 
37 void pack_owned_verify( CommAll & all , const BulkData & mesh );
38 
39 bool unpack_not_owned_verify( CommAll & comm_all ,
40  const BulkData & mesh ,
41  std::ostream & error_log );
42 
43 }
44 
45 bool comm_mesh_verify_parallel_consistency(
46  BulkData & M , std::ostream & error_log )
47 {
48  int result = 1 ;
49 
50  // Verify consistency of parallel attributes
51 
52  result = verify_parallel_attributes( M , error_log );
53 
54  if (M.parallel_size() > 1) {
55  all_reduce( M.parallel() , ReduceMin<1>( & result ) );
56  }
57 
58  // Verify entities against owner.
59 
60  if ( result ) {
61  CommAll all( M.parallel() );
62 
63  pack_owned_verify( all , M );
64 
65  all.allocate_buffers( all.parallel_size() / 4 );
66 
67  pack_owned_verify( all , M );
68 
69  all.communicate();
70 
71  result = unpack_not_owned_verify( all , M , error_log );
72 
73  if (M.parallel_size() > 1) {
74  all_reduce( M.parallel() , ReduceMin<1>( & result ) );
75  }
76  }
77 
78  return result == 1 ;
79 }
80 
81 namespace {
82 
83 bool ordered_comm( const Entity & entity )
84 {
85  const PairIterEntityComm ec = entity.comm();
86  const size_t n = ec.size();
87  for ( size_t i = 1 ; i < n ; ++i ) {
88  if ( ! ( ec[i-1] < ec[i] ) ) {
89  return false ;
90  }
91  }
92  return true ;
93 }
94 
95 bool verify_parallel_attributes( BulkData & M , std::ostream & error_log )
96 {
97  bool result = true ;
98 
99  const MetaData & S = MetaData::get(M);
100  Part & owns_part = S.locally_owned_part();
101  Part & shares_part = S.globally_shared_part();
102 
103  const unsigned p_rank = M.parallel_rank();
104 
105  const size_t EntityRankEnd = MetaData::get(M).entity_rank_count();
106 
107  size_t comm_count = 0 ;
108 
109  for ( size_t itype = 0 ; itype < EntityRankEnd ; ++itype ) {
110  const std::vector< Bucket * > & all_buckets = M.buckets( itype );
111 
112  const std::vector<Bucket*>::const_iterator i_end = all_buckets.end();
113  std::vector<Bucket*>::const_iterator i = all_buckets.begin();
114 
115  while ( i != i_end ) {
116  Bucket & bucket = **i ; ++i ;
117 
118  const bool has_owns_part = has_superset( bucket , owns_part );
119  const bool has_shares_part = has_superset( bucket , shares_part );
120 
121  const Bucket::iterator j_end = bucket.end();
122  Bucket::iterator j = bucket.begin();
123 
124  while ( j != j_end ) {
125  Entity & entity = *j ; ++j ;
126 
127  bool this_result = true ;
128 
129  const unsigned p_owner = entity.owner_rank();
130  const bool ordered = ordered_comm( entity );
131  const bool shares = in_shared( entity );
132  const bool recv_ghost = in_receive_ghost( entity );
133  const bool send_ghost = in_send_ghost( entity );
134  const bool owned_closure = in_owned_closure( entity , p_rank );
135 
136  if ( ! ordered ) {
137  error_log << "Problem is unordered" << std::endl;
138  this_result = false ;
139  }
140 
141  // Owner consistency:
142 
143  if ( has_owns_part && p_owner != p_rank ) {
144  error_log << "problem is owner-consistency check 1: "
145  << "has_owns_part: " << has_owns_part << ", "
146  << "p_owner: " << p_owner << ", "
147  << "p_rank: " << p_rank << std::endl;
148  this_result = false ;
149  }
150 
151  if ( ! has_owns_part && p_owner == p_rank ) {
152  error_log << "problem is owner-consistency check 2: "
153  << "has_owns_part: " << has_owns_part << ", "
154  << "p_owner: " << p_owner << ", "
155  << "p_rank: " << p_rank << std::endl;
156  this_result = false ;
157  }
158 
159  if ( has_shares_part != shares ) {
160  error_log << "problem is owner-consistency check 3: "
161  << "has_shares_part: " << has_shares_part << ", "
162  << "shares: " << shares << std::endl;
163  this_result = false ;
164  }
165 
166  // Definition of 'closure'
167 
168  if ( ( has_owns_part || has_shares_part ) != owned_closure ) {
169  error_log << "problem is closure check 1: "
170  << "has_owns_part: " << has_owns_part << ", "
171  << "has_shares_part: " << has_shares_part << ", "
172  << "owned_closure: " << owned_closure << std::endl;
173  this_result = false ;
174  }
175 
176  // Must be either owned_closure or recv_ghost but not both.
177 
178  if ( owned_closure && recv_ghost ) {
179  error_log << "problem is recv ghost check 1: "
180  << "owned_closure: " << owned_closure << ", "
181  << "recv_ghost: " << recv_ghost << std::endl;
182  this_result = false ;
183  }
184  if ( ! owned_closure && ! recv_ghost ) {
185  error_log << "problem is recv ghost check 2: "
186  << "owned_closure: " << owned_closure << ", "
187  << "recv_ghost: " << recv_ghost << std::endl;
188  this_result = false ;
189  }
190 
191  // If sending as a ghost then I must own it
192 
193  if ( ! has_owns_part && send_ghost ) {
194  error_log << "problem is send ghost check 1: "
195  << "has_owns_part: " << has_owns_part << ", "
196  << "send_ghost: " << send_ghost << std::endl;
197  this_result = false ;
198  }
199 
200  // If shared then I am owner or owner is in the shared list
201 
202  if ( shares && p_owner != p_rank ) {
203  PairIterEntityComm ip = entity.sharing();
204  for ( ; ! ip.empty() && p_owner != ip->proc ; ++ip );
205  if ( ip.empty() ) {
206  error_log << "problem is shared check 1" << std::endl;
207  this_result = false ;
208  }
209  }
210 
211  if ( shares || recv_ghost || send_ghost ) { ++comm_count ; }
212 
213  if ( ! this_result ) {
214  result = false ;
215  error_log << "P" << M.parallel_rank() << ": " ;
216  print_entity_key( error_log , MetaData::get(M), entity.key() );
217  error_log << " ERROR: owner(" << p_owner
218  << ") owns(" << has_owns_part
219  << ") shares(" << has_shares_part
220  << ") owned_closure(" << owned_closure
221  << ") recv_ghost(" << recv_ghost
222  << ") send_ghost(" << send_ghost
223  << ") comm(" ;
224  PairIterEntityComm ip = entity.comm();
225  for ( ; ! ip.empty() ; ++ip ) {
226  error_log << " " << ip->ghost_id << ":" << ip->proc ;
227  }
228  error_log << " )" << std::endl ;
229  }
230  }
231  }
232  }
233 
234  for ( std::vector<Entity*>::const_iterator
235  i = M.entity_comm().begin() ;
236  i != M.entity_comm().end() ; ++i ) {
237 
238  const PairIterEntityComm ec = (*i)->comm();
239 
240  if ( ec.empty() ) {
241  print_entity_key( error_log , MetaData::get(M), (*i)->key() );
242  error_log << " ERROR: in entity_comm but has no comm info" << std::endl ;
243  result = false ;
244  }
245  }
246 
247  if ( M.entity_comm().size() != comm_count ) {
248  error_log << " ERROR: entity_comm.size() = " << M.entity_comm().size();
249  error_log << " != " << comm_count << " = entities with comm info" ;
250  error_log << std::endl ;
251  result = false ;
252  }
253 
254  return result ;
255 }
256 
257 //----------------------------------------------------------------------------
258 // Packing my owned entities.
259 
260 void insert( std::vector<unsigned> & vec , unsigned val )
261 {
262  std::vector<unsigned>::iterator j =
263  std::lower_bound( vec.begin() , vec.end() , val );
264  if ( j == vec.end() || *j != val ) {
265  vec.insert( j , val );
266  }
267 }
268 
269 void pack_owned_verify( CommAll & all , const BulkData & mesh )
270 {
271  const std::vector<Entity*> & entity_comm = mesh.entity_comm();
272  const unsigned p_rank = all.parallel_rank();
273 
274  for ( std::vector<Entity*>::const_iterator
275  i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
276 
277  Entity & entity = **i ;
278 
279  if ( entity.owner_rank() == p_rank ) {
280 
281  std::vector<unsigned> share_proc ;
282  std::vector<unsigned> ghost_proc ;
283 
284  const PairIterEntityComm comm = entity.comm();
285 
286  for ( size_t j = 0 ; j < comm.size() ; ++j ) {
287  if ( comm[j].ghost_id == 0 ) {
288  // Will be ordered by proc
289  share_proc.push_back( comm[j].proc );
290  }
291  else {
292  // No guarantee of ordering by proc
293  insert( ghost_proc , comm[j].proc );
294  }
295  }
296 
297  const unsigned share_count = share_proc.size();
298 
299  for ( size_t j = 0 ; j < share_proc.size() ; ++j ) {
300 
301  // Sharing process, send sharing process list
302 
303  const unsigned p = share_proc[j] ;
304 
305  CommBuffer & buf = all.send_buffer( p );
306 
307  pack_entity_info( buf , entity );
308 
309  buf.pack<unsigned>( share_count );
310 
311  // Pack what the receiver should have:
312  // My list, remove receiver, add myself
313  size_t k = 0 ;
314  for ( ; k < share_count && share_proc[k] < p_rank ; ++k ) {
315  if ( k != j ) { buf.pack<unsigned>( share_proc[k] ); }
316  }
317  buf.pack<unsigned>( p_rank );
318  for ( ; k < share_count ; ++k ) {
319  if ( k != j ) { buf.pack<unsigned>( share_proc[k] ); }
320  }
321  }
322 
323  for ( size_t j = 0 ; j < ghost_proc.size() ; ++j ) {
324  const unsigned p = ghost_proc[j] ;
325 
326  CommBuffer & buf = all.send_buffer( p );
327 
328  pack_entity_info( buf , entity );
329 
330  // What ghost subsets go to this process?
331  unsigned count = 0 ;
332  for ( size_t k = 0 ; k < comm.size() ; ++k ) {
333  if ( comm[k].ghost_id != 0 && comm[k].proc == p ) {
334  ++count ;
335  }
336  }
337  buf.pack<unsigned>( count );
338  for ( size_t k = 0 ; k < comm.size() ; ++k ) {
339  if ( comm[k].ghost_id != 0 && comm[k].proc == p ) {
340  buf.pack<unsigned>( comm[k].ghost_id );
341  }
342  }
343  }
344  }
345  }
346 }
347 
348 //----------------------------------------------------------------------------
349 // Unpacking all of my not-owned entities.
350 
351 bool unpack_not_owned_verify( CommAll & comm_all ,
352  const BulkData & mesh ,
353  std::ostream & error_log )
354 {
355  const MetaData & meta = MetaData::get(mesh);
356  Part * const owns_part = & meta.locally_owned_part();
357  Part * const shares_part = & meta.globally_shared_part();
358  const PartVector & mesh_parts = meta.get_parts();
359  const unsigned p_rank = mesh.parallel_rank();
360  const std::vector<Entity*> & entity_comm = mesh.entity_comm();
361 
362  bool result = true ;
363 
364  EntityKey recv_entity_key ;
365  unsigned recv_owner_rank = 0 ;
366  unsigned recv_comm_count = 0 ;
367  std::vector<Part*> recv_parts ;
368  std::vector<Relation> recv_relations ;
369  std::vector<unsigned> recv_comm ;
370 
371  for ( std::vector<Entity*>::const_iterator
372  i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
373 
374  Entity & entity = **i ;
375 
376  if ( entity.owner_rank() != p_rank ) {
377 
378  const Bucket & bucket = entity.bucket();
379 
380  std::pair<const unsigned *,const unsigned *>
381  part_ordinals = bucket.superset_part_ordinals();
382 
383  CommBuffer & buf = comm_all.recv_buffer( entity.owner_rank() );
384 
385  unpack_entity_info( buf , mesh ,
386  recv_entity_key , recv_owner_rank ,
387  recv_parts , recv_relations );
388 
389  recv_comm_count = 0 ;
390  buf.unpack<unsigned>( recv_comm_count );
391  recv_comm.resize( recv_comm_count );
392  buf.unpack<unsigned>( & recv_comm[0] , recv_comm_count );
393 
394  // Match key and owner
395 
396  const bool bad_key = entity.key() != recv_entity_key ;
397  const bool bad_own = entity.owner_rank() != recv_owner_rank ;
398  bool bad_part = false ;
399  bool bad_rel = false ;
400  bool bad_comm = false ;
401 
402  // Compare communication information:
403 
404  if ( ! bad_key && ! bad_own ) {
405  const PairIterEntityComm ec = entity.comm();
406  const unsigned ec_size = ec.size();
407  bad_comm = ec_size != recv_comm.size();
408  if ( ! bad_comm ) {
409  size_t j = 0 ;
410  if ( in_shared( entity ) ) {
411  for ( ; j < ec_size &&
412  ec[j].ghost_id == 0 &&
413  ec[j].proc == recv_comm[j] ; ++j );
414  bad_comm = j != ec_size ;
415  }
416  else {
417  for ( ; j < ec_size &&
418  ec[j].ghost_id == recv_comm[j] &&
419  ec[j].proc == entity.owner_rank() ; ++j );
420  bad_comm = j != ec_size ;
421  }
422  }
423  }
424 
425  // Compare everything but the owns part and uses part
426 
427  if ( ! bad_key && ! bad_own && ! bad_comm ) {
428 
429  const unsigned * k = part_ordinals.first ;
430 
431  std::vector<Part*>::iterator ip = recv_parts.begin();
432 
433  for ( ; ! bad_part && ip != recv_parts.end() ; ++ip ) {
434  if ( owns_part != *ip ) {
435  if ( shares_part != *ip ) {
436  // All not-owned and not-shares parts must match:
437  bad_part = k == part_ordinals.second ||
438  (*ip)->mesh_meta_data_ordinal() != *k ;
439  ++k ;
440  }
441  else if ( k != part_ordinals.second &&
442  *k == shares_part->mesh_meta_data_ordinal() ) {
443  // shares-part matches
444  ++k ;
445  }
446  }
447  }
448  }
449 
450  // Compare the closure relations:
451  if ( ! bad_key && ! bad_own && ! bad_comm && ! bad_part ) {
452 
453  PairIterRelation ir = entity.relations();
454 
455  std::vector<Relation>::iterator jr = recv_relations.begin() ;
456 
457  for ( ; ! bad_rel && jr != recv_relations.end() &&
458  jr->entity_rank() < entity.entity_rank() ; ++jr , ++ir ) {
459  bad_rel = ir.empty() || *jr != *ir ;
460  }
461  }
462 
463  // The rest of this code is just error handling
464  if ( bad_key || bad_own || bad_comm || bad_part || bad_rel ) {
465  error_log << "P" << p_rank << ": " ;
466  print_entity_key( error_log , meta, entity.key() );
467  error_log << " owner(" << entity.owner_rank() << ")" ;
468 
469  if ( bad_key || bad_own ) {
470  error_log << " != received " ;
471  print_entity_key( error_log , meta, recv_entity_key );
472  error_log << " owner(" << recv_owner_rank
473  << ")" << std::endl ;
474  }
475  else if ( bad_comm ) {
476  const PairIterEntityComm ec = entity.comm();
477  if ( in_shared( entity ) ) {
478  error_log << " sharing(" ;
479  for ( size_t j = 0 ; j < ec.size() &&
480  ec[j].ghost_id == 0 ; ++j ) {
481  error_log << " " << ec[j].proc ;
482  }
483  error_log << " ) != received sharing(" ;
484  for ( size_t j = 0 ; j < recv_comm.size() ; ++j ) {
485  error_log << " " << recv_comm[j] ;
486  }
487  error_log << " )" << std::endl ;
488  }
489  else {
490  error_log << " ghosting(" ;
491  for ( size_t j = 0 ; j < ec.size() ; ++j ) {
492  error_log << " (g" << ec[j].ghost_id ;
493  error_log << ",p" << ec[j].proc ;
494  error_log << ")" ;
495  }
496  error_log << " ) != received ghosting(" ;
497  for ( size_t j = 0 ; j < recv_comm.size() ; ++j ) {
498  error_log << " (g" << recv_comm[j] ;
499  error_log << ",p" << entity.owner_rank();
500  error_log << ")" ;
501  }
502  error_log << " )" << std::endl ;
503  }
504  }
505  else if ( bad_part ) {
506  error_log << " Parts( " ;
507 
508  for ( const unsigned * k = part_ordinals.first ;
509  k < part_ordinals.second ; ++k ) {
510  error_log << " \"" << mesh_parts[ *k ]->name() << "\"" ;
511  }
512  error_log << " ) != received Parts( " ;
513 
514  for ( std::vector<Part*>::iterator
515  ip = recv_parts.begin();
516  ip != recv_parts.end() ; ++ip ) {
517  error_log << " \"" << (*ip)->name() << "\"" ;
518  }
519  error_log << " )" << std::endl ;
520  }
521  else if ( bad_rel ) {
522  error_log << " Relations(" ;
523  PairIterRelation ir = entity.relations();
524  for ( ; ! ir.empty() &&
525  ir->entity_rank() < entity.entity_rank() ; ++ir ) {
526  error_log << " " << *ir ;
527  }
528  error_log << " ) != received Relations(" ;
529  std::vector<Relation>::iterator jr = recv_relations.begin() ;
530  for ( ; jr != recv_relations.end() &&
531  jr->entity_rank() < entity.entity_rank() ; ++jr ) {
532  error_log << " " << *jr ;
533  }
534  error_log << " )" << std::endl ;
535  }
536  result = false ;
537  }
538  }
539  }
540 
541  return result ;
542 }
543 
544 } // namespace<>
545 
546 } // namespace mesh
547 } // namespace stk_classic
548 
PairIterEntityComm comm() const
Complete communicaiton list for this entity.
Definition: Entity.hpp:181
bool has_superset(const Bucket &bucket, const unsigned &ordinal)
Is this bucket a subset of the given part by partID.
Definition: Bucket.cpp:127
PairIterEntityComm sharing() const
Parallel processes which share this entity.
Definition: Entity.hpp:178
Bucket & bucket() const
The bucket which holds this mesh entity&#39;s field data.
Definition: Entity.hpp:141
const EntityKey & key() const
The globally unique key ( entity type + identifier ) of this entity.
Definition: Entity.hpp:138
std::pair< const unsigned *, const unsigned * > superset_part_ordinals() const
Definition: Bucket.hpp:188
PairIterRelation relations() const
All Entity relations for which this entity is a member. The relations are ordered from lowest entity-...
Definition: Entity.hpp:161
Sierra Toolkit.
EntityRank entity_rank() const
The rank of this entity.
Definition: Entity.hpp:128
std::vector< Part *> PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
Definition: Types.hpp:31
PairIter< std::vector< EntityCommInfo >::const_iterator > PairIterEntityComm
Span of ( communication-subset-ordinal , process-rank ) pairs for the communication of an entity...
Definition: Types.hpp:128
unsigned owner_rank() const
Parallel processor rank of the processor which owns this entity.
Definition: Entity.hpp:175
bool insert(PartVector &v, Part &part)
Insert a part into a properly ordered collection of parts. Returns true if this is a new insertion...
Definition: Part.cpp:85
eastl::iterator_traits< InputIterator >::difference_type count(InputIterator first, InputIterator last, const T &value)