Panzer  Version of the Day
Panzer_ScatterResidual_BlockedEpetra_Hessian_impl.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Panzer: A partial differential equation assembly
5 // engine for strongly coupled complex multiphysics systems
6 // Copyright (2011) Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and
39 // Eric C. Cyr (eccyr@sandia.gov)
40 // ***********************************************************************
41 // @HEADER
42 //
43 #ifndef __Panzer_ScatterResidual_BlockedEpetra_Hessian_impl_hpp__
44 #define __Panzer_ScatterResidual_BlockedEpetra_Hessian_impl_hpp__
45 
46 // only do this if required by the user
47 #ifdef Panzer_BUILD_HESSIAN_SUPPORT
48 
49 // the includes for this file come in as a result of the includes in the main
50 // Epetra scatter residual file
51 
52 namespace panzer {
53 
54 // **************************************************************
55 // Hessian Specialization
56 // **************************************************************
57 template<typename TRAITS,typename LO,typename GO>
59 ScatterResidual_BlockedEpetra(const std::vector<Teuchos::RCP<const UniqueGlobalIndexer<LO,int> > > & rIndexers,
60  const std::vector<Teuchos::RCP<const UniqueGlobalIndexer<LO,int> > > & cIndexers,
61  const Teuchos::ParameterList& p,
62  bool useDiscreteAdjoint)
63  : rowIndexers_(rIndexers)
64  , colIndexers_(cIndexers)
65  , globalDataKey_("Residual Scatter Container")
66  , useDiscreteAdjoint_(useDiscreteAdjoint)
67 {
68  std::string scatterName = p.get<std::string>("Scatter Name");
69  scatterHolder_ =
70  Teuchos::rcp(new PHX::Tag<ScalarT>(scatterName,Teuchos::rcp(new PHX::MDALayout<Dummy>(0))));
71 
72  // get names to be evaluated
73  const std::vector<std::string>& names =
74  *(p.get< Teuchos::RCP< std::vector<std::string> > >("Dependent Names"));
75 
76  // grab map from evaluated names to field names
77  fieldMap_ = p.get< Teuchos::RCP< std::map<std::string,std::string> > >("Dependent Map");
78 
79  Teuchos::RCP<PHX::DataLayout> dl =
80  p.get< Teuchos::RCP<const panzer::PureBasis> >("Basis")->functional;
81 
82  // build the vector of fields that this is dependent on
83  scatterFields_.resize(names.size());
84  for (std::size_t eq = 0; eq < names.size(); ++eq) {
85  scatterFields_[eq] = PHX::MDField<const ScalarT,Cell,NODE>(names[eq],dl);
86 
87  // tell the field manager that we depend on this field
88  this->addDependentField(scatterFields_[eq]);
89  }
90 
91  // this is what this evaluator provides
92  this->addEvaluatedField(*scatterHolder_);
93 
94  if (p.isType<std::string>("Global Data Key"))
95  globalDataKey_ = p.get<std::string>("Global Data Key");
96  if (p.isType<bool>("Use Discrete Adjoint"))
97  useDiscreteAdjoint = p.get<bool>("Use Discrete Adjoint");
98 
99  // discrete adjoint does not work with non-square matrices
100  if(useDiscreteAdjoint)
101  { TEUCHOS_ASSERT(colIndexers_.size()==0); }
102 
103  if(colIndexers_.size()==0)
104  colIndexers_ = rowIndexers_;
105 
106  this->setName(scatterName+" Scatter Residual BlockedEpetra (Hessian)");
107 }
108 
109 template<typename TRAITS,typename LO,typename GO>
110 void
112 postRegistrationSetup(typename TRAITS::SetupData d,
114 {
115  indexerIds_.resize(scatterFields_.size());
116  subFieldIds_.resize(scatterFields_.size());
117 
118  // load required field numbers for fast use
119  for(std::size_t fd=0;fd<scatterFields_.size();++fd) {
120  // get field ID from DOF manager
121  std::string fieldName = fieldMap_->find(scatterFields_[fd].fieldTag().name())->second;
122 
123  indexerIds_[fd] = getFieldBlock(fieldName,rowIndexers_);
124  subFieldIds_[fd] = rowIndexers_[indexerIds_[fd]]->getFieldNum(fieldName);
125 
126  // fill field data object
127  this->utils.setFieldData(scatterFields_[fd],fm);
128  }
129 }
130 
131 template<typename TRAITS,typename LO,typename GO>
132 void
133 ScatterResidual_BlockedEpetra<panzer::Traits::Hessian,TRAITS,LO,GO>::
134 preEvaluate(typename TRAITS::PreEvalData d)
135 {
136  using Teuchos::RCP;
137  using Teuchos::rcp_dynamic_cast;
138 
139  typedef BlockedEpetraLinearObjContainer BLOC;
140  typedef BlockedEpetraLinearObjContainer ELOC;
141 
142  // extract linear object container
143  RCP<const BLOC> blockedContainer = rcp_dynamic_cast<const BLOC>(d.gedc.getDataObject(globalDataKey_));
144  RCP<const ELOC> epetraContainer = rcp_dynamic_cast<const ELOC>(d.gedc.getDataObject(globalDataKey_));
145 
146  // if its blocked do this
147  if(blockedContainer!=Teuchos::null) {
148  Jac_ = rcp_dynamic_cast<Thyra::BlockedLinearOpBase<double> >(blockedContainer->get_A());
149  }
150  else if(epetraContainer!=Teuchos::null) {
151  // convert it into a blocked operator
152  RCP<Thyra::LinearOpBase<double> > J = blockedContainer->get_A_th();
153  Jac_ = rcp_dynamic_cast<Thyra::BlockedLinearOpBase<double> >(Thyra::nonconstBlock1x1(J));
154  }
155 
156  TEUCHOS_ASSERT(Jac_!=Teuchos::null);
157 }
158 
159 template<typename TRAITS,typename LO,typename GO>
160 void
162 evaluateFields(typename TRAITS::EvalData workset)
163 {
164  using Teuchos::RCP;
165  using Teuchos::ArrayRCP;
166  using Teuchos::ptrFromRef;
167  using Teuchos::rcp_dynamic_cast;
168 
169  using Thyra::VectorBase;
170  using Thyra::SpmdVectorBase;
173 
174  std::vector<double> jacRow;
175 
176  // for convenience pull out some objects from workset
177  std::string blockId = this->wda(workset).block_id;
178  const std::vector<std::size_t> & localCellIds = this->wda(workset).cell_local_ids;
179 
180  int numFieldBlocks = Teuchos::as<int>(colIndexers_.size());
181 
182  std::vector<int> blockOffsets;
183  computeBlockOffsets(blockId,colIndexers_,blockOffsets);
184 
185  std::unordered_map<std::pair<int,int>,Teuchos::RCP<Epetra_CrsMatrix>,panzer::pair_hash> jacEpetraBlocks;
186 
187  // loop over each field to be scattered
188  for(std::size_t fieldIndex = 0; fieldIndex < scatterFields_.size(); fieldIndex++) {
189  int rowIndexer = indexerIds_[fieldIndex];
190  int subFieldNum = subFieldIds_[fieldIndex];
191 
192  auto subRowIndexer = rowIndexers_[rowIndexer];
193  const std::vector<int> & elmtOffset = subRowIndexer->getGIDFieldOffsets(blockId,subFieldNum);
194 
195  // scatter operation for each cell in workset
196  for(std::size_t worksetCellIndex=0;worksetCellIndex<localCellIds.size();++worksetCellIndex) {
197  std::size_t cellLocalId = localCellIds[worksetCellIndex];
198 
199  const std::vector<LO> & rLIDs = subRowIndexer->getElementLIDs(cellLocalId);
200 
201  // loop over the basis functions (currently they are nodes)
202  for(std::size_t rowBasisNum = 0; rowBasisNum < elmtOffset.size(); rowBasisNum++) {
203  const ScalarT scatterField = (scatterFields_[fieldIndex])(worksetCellIndex,rowBasisNum);
204  int rowOffset = elmtOffset[rowBasisNum];
205  int r_lid = rLIDs[rowOffset];
206 
207  // loop over the sensitivity indices: all DOFs on a cell
208  jacRow.resize(scatterField.size());
209 
210  // For Neumann conditions with no dependence on degrees of freedom, there should be no Jacobian contribution
211  if(scatterField.size() == 0)
212  continue;
213 
214  for(int sensIndex=0;sensIndex<scatterField.size();++sensIndex)
215  jacRow[sensIndex] = scatterField.fastAccessDx(sensIndex).fastAccessDx(0);
216 
217  // scatter the row to each block
218  for(int colIndexer=0;colIndexer<numFieldBlocks;colIndexer++) {
219  int start = blockOffsets[colIndexer];
220  int end = blockOffsets[colIndexer+1];
221 
222  if(end-start<=0)
223  continue;
224 
225  auto subColIndexer = colIndexers_[colIndexer];
226  const std::vector<LO> & cLIDs = subColIndexer->getElementLIDs(cellLocalId);
227 
228  TEUCHOS_ASSERT(end-start==Teuchos::as<int>(cLIDs.size()));
229 
230  // check hash table for jacobian sub block
231  std::pair<int,int> blockIndex = std::make_pair(rowIndexer,colIndexer);
232  Teuchos::RCP<Epetra_CrsMatrix> subJac = jacEpetraBlocks[blockIndex];
233 
234  // if you didn't find one before, add it to the hash table
235  if(subJac==Teuchos::null) {
236  Teuchos::RCP<Thyra::LinearOpBase<double> > tOp = Jac_->getNonconstBlock(blockIndex.first,blockIndex.second);
237 
238  // block operator is null, don't do anything (it is excluded)
239  if(Teuchos::is_null(tOp))
240  continue;
241 
242  Teuchos::RCP<Epetra_Operator> eOp = Thyra::get_Epetra_Operator(*tOp);
243  subJac = rcp_dynamic_cast<Epetra_CrsMatrix>(eOp,true);
244  jacEpetraBlocks[blockIndex] = subJac;
245  }
246 
247  // Sum Jacobian
248  {
249  int err = subJac->SumIntoMyValues(r_lid, end-start, &jacRow[start],&cLIDs[0]);
250  if(err!=0) {
251 
252  std::stringstream ss;
253  ss << "Failed inserting row: " << "LID = " << r_lid << ": ";
254  for(int i=0;i<end-start;i++)
255  ss << cLIDs[i] << " ";
256  ss << std::endl;
257  ss << "Into block " << rowIndexer << ", " << colIndexer << std::endl;
258 
259  ss << "scatter field = ";
260  scatterFields_[fieldIndex].print(ss);
261  ss << std::endl;
262 
263  ss << "values = ";
264  for(int i=start;i<end;i++)
265  ss << jacRow[i] << " ";
266  ss << std::endl;
267 
268  std::cout << ss.str() << std::endl;
269 
270  TEUCHOS_TEST_FOR_EXCEPTION(err!=0,std::runtime_error,ss.str());
271  }
272  }
273  }
274  } // end rowBasisNum
275  } // end fieldIndex
276  }
277 }
278 
279 }
280 
281 // **************************************************************
282 #endif
283 
284 #endif
int getFieldBlock(const std::string &fieldName, const std::vector< Teuchos::RCP< UniqueGlobalIndexer< LocalOrdinalT, GlobalOrdinalT > > > &ugis)
std::vector< PHX::MDField< const ScalarT, panzer::Cell, panzer::Point > > scatterFields_
void computeBlockOffsets(const std::string &blockId, const std::vector< Teuchos::RCP< UniqueGlobalIndexer< LocalOrdinalT, GlobalOrdinalT > > > &ugis, std::vector< int > &blockOffsets)
PHX::MDField< ScalarT > vector
void postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager< TRAITS > &vm)