Stokhos Package Browser (Single Doxygen Collection)  Version of the Day
MPAssembly/TestAssembly.cpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Stokhos Package
5 // Copyright (2009) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Eric T. Phipps (etphipp@sandia.gov).
38 //
39 // ***********************************************************************
40 // @HEADER
41 
42 #include <iostream>
43 
44 // Tests
45 #include "TestAssembly.hpp"
46 
47 // Devices
48 #include "Kokkos_Core.hpp"
49 
50 // Utilities
51 #include "Teuchos_DefaultComm.hpp"
52 #include "Teuchos_CommandLineProcessor.hpp"
53 #include "Teuchos_StandardCatchMacros.hpp"
54 #ifdef KOKKOS_HAVE_CUDA
55 #include "cuda_runtime_api.h"
56 #endif
57 
58 template <typename Storage>
59 void mainHost(const Teuchos::RCP<const Teuchos::Comm<int> >& comm ,
60  const int use_print ,
61  const int use_trials ,
62  const int use_nodes[] ,
63  const bool check ,
65 #ifdef __MIC__
66  const int entry_min = 8;
67  const int entry_max = 48;
68  const int entry_step = 8;
69 #else
70  const int entry_min = 4;
71  const int entry_max = 32;
72  const int entry_step = 4;
73  // const int entry_min = 16;
74  // const int entry_max = 16;
75  // const int entry_step = 16;
76 #endif
77 
78  performance_test_driver<Storage,entry_min,entry_max,entry_step>(
79  comm, use_print, use_trials, use_nodes, check, dev_config);
80 }
81 
82 template <typename Storage>
83 void mainCuda(const Teuchos::RCP<const Teuchos::Comm<int> >& comm ,
84  const int use_print ,
85  const int use_trials ,
86  const int use_nodes[] ,
87  const bool check ,
89  const int entry_min = 16;
90  const int entry_max = 64;
91  const int entry_step = 16;
92  performance_test_driver<Storage,entry_min,entry_max,entry_step>(
93  comm, use_print, use_trials, use_nodes, check, dev_config);
94 }
95 
96 int main(int argc, char *argv[])
97 {
98  bool success = true;
99  bool verbose = false;
100  try {
101 
102  Teuchos::oblackholestream blackHole;
103  Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
104 
105  Teuchos::RCP<const Teuchos::Comm<int> > comm =
106  Teuchos::DefaultComm<int>::getComm();
107 
108  const size_t num_sockets = Kokkos::hwloc::get_available_numa_count();
109  const size_t num_cores_per_socket =
110  Kokkos::hwloc::get_available_cores_per_numa();
111  const size_t num_threads_per_core =
112  Kokkos::hwloc::get_available_threads_per_core();
113 
114  // Setup command line options
115  Teuchos::CommandLineProcessor CLP;
116  CLP.setDocString(
117  "This test performance of MP::Vector FEM assembly.\n");
118  int nGrid = 32;
119  CLP.setOption("n", &nGrid, "Number of mesh points in the each direction");
120  int nIter = 10;
121  CLP.setOption("ni", &nIter, "Number of assembly iterations");
122  bool print = false;
123  CLP.setOption("print", "no-print", &print, "Print debugging output");
124  bool check = false;
125  int num_cores = num_cores_per_socket * num_sockets;
126  CLP.setOption("cores", &num_cores,
127  "Number of CPU cores to use (defaults to all)");
128  int num_hyper_threads = num_threads_per_core;
129  CLP.setOption("hyperthreads", &num_hyper_threads,
130  "Number of hyper threads per core to use (defaults to all)");
131  int threads_per_vector = 1;
132  CLP.setOption("threads_per_vector", &threads_per_vector,
133  "Number of threads to use within each vector");
134  CLP.setOption("check", "no-check", &check, "Check correctness");
135 #ifdef KOKKOS_HAVE_SERIAL
136  bool serial = true;
137  CLP.setOption("serial", "no-serial", &serial, "Enable Serial device");
138 #endif
139 #ifdef KOKKOS_HAVE_PTHREAD
140  bool threads = true;
141  CLP.setOption("threads", "no-threads", &threads, "Enable Threads device");
142 #endif
143 #ifdef KOKKOS_HAVE_OPENMP
144  bool openmp = true;
145  CLP.setOption("openmp", "no-openmp", &openmp, "Enable OpenMP device");
146 #endif
147 #ifdef KOKKOS_HAVE_CUDA
148  bool cuda = true;
149  CLP.setOption("cuda", "no-cuda", &cuda, "Enable Cuda device");
150  int cuda_threads_per_vector = 16;
151  CLP.setOption("cuda_threads_per_vector", &cuda_threads_per_vector,
152  "Number of Cuda threads to use within each vector");
153  int cuda_block_size = 256;
154  CLP.setOption("cuda_block_size", &cuda_block_size,
155  "Cuda block size");
156  int num_cuda_blocks = 0;
157  CLP.setOption("num_cuda_blocks", &num_cuda_blocks,
158  "Number of Cuda blocks (0 implies the default choice)");
159  int device_id = -1;
160  CLP.setOption("device", &device_id, "CUDA device ID. Set to default of -1 to use the default device as determined by the local node MPI rank and --ngpus");
161  int ngpus = 1;
162  CLP.setOption("ngpus", &ngpus, "Number of GPUs per node for multi-GPU runs via MPI");
163 #endif
164  CLP.parse( argc, argv );
165 
166  int use_nodes[3];
167  use_nodes[0] = nGrid; use_nodes[1] = nGrid; use_nodes[2] = nGrid;
168 
169  typedef int Ordinal;
170  typedef double Scalar;
171 
172 #ifdef KOKKOS_HAVE_SERIAL
173  if (serial) {
174  typedef Kokkos::Serial Device;
176 
177  Kokkos::Serial::initialize();
178 
179  if (comm->getRank() == 0)
180  std::cout << std::endl
181  << "Serial performance with " << comm->getSize()
182  << " MPI ranks" << std::endl;
183 
184  Kokkos::Example::FENL::DeviceConfig dev_config(1, 1, 1);
185 
186  mainHost<Storage>(comm, print, nIter, use_nodes, check,
187  dev_config);
188 
189  Kokkos::Serial::finalize();
190  }
191 #endif
192 
193 #ifdef KOKKOS_HAVE_PTHREAD
194  if (threads) {
195  typedef Kokkos::Threads Device;
197 
198  Kokkos::Threads::initialize(num_cores*num_hyper_threads);
199 
200  if (comm->getRank() == 0)
201  std::cout << std::endl
202  << "Threads performance with " << comm->getSize()
203  << " MPI ranks and " << num_cores*num_hyper_threads
204  << " threads per rank:" << std::endl;
205 
207  threads_per_vector,
208  num_hyper_threads / threads_per_vector);
209 
210  mainHost<Storage>(comm, print, nIter, use_nodes, check,
211  dev_config);
212 
213  Kokkos::Threads::finalize();
214  }
215 #endif
216 
217 #ifdef KOKKOS_HAVE_OPENMP
218  if (openmp) {
219  typedef Kokkos::OpenMP Device;
221 
222  Kokkos::OpenMP::initialize(num_cores*num_hyper_threads);
223 
224  if (comm->getRank() == 0)
225  std::cout << std::endl
226  << "OpenMP performance with " << comm->getSize()
227  << " MPI ranks and " << num_cores*num_hyper_threads
228  << " threads per rank:" << std::endl;
229 
231  threads_per_vector,
232  num_hyper_threads / threads_per_vector);
233 
234  mainHost<Storage>(comm, print, nIter, use_nodes, check,
235  dev_config);
236 
237  Kokkos::OpenMP::finalize();
238  }
239 #endif
240 
241 #ifdef KOKKOS_HAVE_CUDA
242  if (cuda) {
243  typedef Kokkos::Cuda Device;
245 
246  if (device_id == -1) {
247  int local_rank = 0;
248  char *str;
249  if ((str = std::getenv("SLURM_LOCALID")))
250  local_rank = std::atoi(str);
251  else if ((str = std::getenv("MV2_COMM_WORLD_LOCAL_RANK")))
252  local_rank = std::atoi(str);
253  else if ((str = getenv("OMPI_COMM_WORLD_LOCAL_RANK")))
254  local_rank = std::atoi(str);
255  device_id = local_rank % ngpus;
256 
257  // Check device is valid
258  int num_device; cudaGetDeviceCount(&num_device);
259  TEUCHOS_TEST_FOR_EXCEPTION(
260  device_id >= num_device, std::logic_error,
261  "Invalid device ID " << device_id << ". You probably are trying" <<
262  " to run with too many GPUs per node");
263  }
264 
265  Kokkos::HostSpace::execution_space::initialize();
266  Kokkos::Cuda::initialize(Kokkos::Cuda::SelectDevice(device_id));
267 
268  cudaDeviceProp deviceProp;
269  cudaGetDeviceProperties(&deviceProp, device_id);
270  if (comm->getRank() == 0)
271  std::cout << std::endl
272  << "CUDA performance performance with " << comm->getSize()
273  << " MPI ranks and device " << device_id << " ("
274  << deviceProp.name << "):"
275  << std::endl;
276 
278  num_cuda_blocks,
279  cuda_threads_per_vector,
280  cuda_threads_per_vector == 0 ? 0 : cuda_block_size / cuda_threads_per_vector);
281 
282  mainCuda<Storage>(comm, print, nIter, use_nodes, check,
283  dev_config);
284 
285  Kokkos::HostSpace::execution_space::finalize();
286  Kokkos::Cuda::finalize();
287  }
288 #endif
289 
290  }
291  TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
292 
293  if (success)
294  return 0;
295  return -1;
296 }
Stokhos::StandardStorage< int, double > Storage
void mainHost(const Teuchos::RCP< const Teuchos::Comm< int > > &comm, const int use_print, const int use_trials, const int use_nodes[], const bool check, Kokkos::Example::FENL::DeviceConfig dev_config)
Statically allocated storage class.
int check(Epetra_CrsGraph &A, int NumMyRows1, int NumGlobalRows1, int NumMyNonzeros1, int NumGlobalNonzeros1, int *MyGlobalElements, bool verbose)
int main(int argc, char *argv[])
void mainCuda(const Teuchos::RCP< const Teuchos::Comm< int > > &comm, const int use_print, const int use_trials, const int use_nodes[], const bool check, Kokkos::Example::FENL::DeviceConfig dev_config)
pce_type Scalar