diff --git a/packages/teuchos/core/src/Teuchos_GlobalMPISession.cpp b/packages/teuchos/core/src/Teuchos_GlobalMPISession.cpp index 25f557dc2706..e0e0d82bfba0 100644 --- a/packages/teuchos/core/src/Teuchos_GlobalMPISession.cpp +++ b/packages/teuchos/core/src/Teuchos_GlobalMPISession.cpp @@ -17,12 +17,6 @@ # include "mpi.h" #endif -#ifdef HAVE_TEUCHOSCORE_KOKKOS -# include "Kokkos_Core.hpp" -#endif // HAVE_TEUCHOSCORE_KOKKOS - - - namespace Teuchos { @@ -31,15 +25,10 @@ bool GlobalMPISession::mpiIsFinalized_ = false; int GlobalMPISession::rank_ = 0 ; int GlobalMPISession::nProc_ = 1 ; -#ifdef HAVE_TEUCHOSCORE_KOKKOS - // We have to invoke the std::vector's constructor here, // because it's a class (static) variable. std::vector GlobalMPISession::argvCopy_; -#endif // HAVE_TEUCHOSCORE_KOKKOS - - GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out ) { std::ostringstream oss; @@ -120,7 +109,6 @@ GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out ) #endif -#ifdef HAVE_TEUCHOSCORE_KOKKOS // mfh 15/16 Apr 2016: This is the one chance we get to save the // command-line arguments, so that we can (later) initialize Kokkos // with the correct number of threads as specified by (e.g.,) the @@ -136,36 +124,23 @@ GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out ) // requirement. const int numArgs = *argc; - argvCopy_.resize (numArgs); + argvCopy_.resize (numArgs); for (int c = 0; c < numArgs; ++c) { argvCopy_[c] = std::string ((*argv)[c]); // deep copy } -#endif // HAVE_TEUCHOSCORE_KOKKOS + + } - -#ifdef HAVE_TEUCHOSCORE_KOKKOS + std::vector GlobalMPISession::getArgv () { return argvCopy_; } -#endif // HAVE_TEUCHOSCORE_KOKKOS - + GlobalMPISession::~GlobalMPISession() { - -#ifdef HAVE_TEUCHOSCORE_KOKKOS - try { - if (Kokkos::is_initialized()) - Kokkos::finalize(); - } - catch (const std::runtime_error& e) { - std::cerr << "Kokkos::finalize failed:\n" - << e.what() << "\n"; - } -#endif - haveMPIState_ = false; #ifdef HAVE_MPI const int mpierr = ::MPI_Finalize(); diff --git a/packages/teuchos/core/src/Teuchos_GlobalMPISession.hpp b/packages/teuchos/core/src/Teuchos_GlobalMPISession.hpp index f7ae8019abe0..bbe08e864f01 100644 --- a/packages/teuchos/core/src/Teuchos_GlobalMPISession.hpp +++ b/packages/teuchos/core/src/Teuchos_GlobalMPISession.hpp @@ -231,7 +231,6 @@ class TEUCHOSCORE_LIB_DLL_EXPORT GlobalMPISession */ static void allGather(int localVal, const ArrayView &allVals); -#ifdef HAVE_TEUCHOSCORE_KOKKOS /// \brief Fetch a deep copy of the input arguments to \c main() /// (that is, \c argv), as given to GlobalMPISession's /// constructor. @@ -240,7 +239,6 @@ class TEUCHOSCORE_LIB_DLL_EXPORT GlobalMPISession /// yet, return an empty vector. Else, return the input /// arguments. static std::vector getArgv (); -#endif // HAVE_TEUCHOSCORE_KOKKOS //@} private: @@ -249,7 +247,7 @@ class TEUCHOSCORE_LIB_DLL_EXPORT GlobalMPISession static bool mpiIsFinalized_; static int rank_; static int nProc_; -#ifdef HAVE_TEUCHOSCORE_KOKKOS + /// \brief Deep copy of the input arguments. /// /// This is useful if we want to call Kokkos::initialize later with @@ -258,7 +256,6 @@ class TEUCHOSCORE_LIB_DLL_EXPORT GlobalMPISession /// after calling this object's constructor. That could mess up /// indexing if we just keep a pointer to the original. static std::vector argvCopy_; -#endif // HAVE_TEUCHOSCORE_KOKKOS static void initialize( std::ostream *out ); diff --git a/packages/tpetra/core/src/Tpetra_Details_initializeKokkos.cpp b/packages/tpetra/core/src/Tpetra_Details_initializeKokkos.cpp index c5081e0ee16a..f977cb583e2c 100644 --- a/packages/tpetra/core/src/Tpetra_Details_initializeKokkos.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_initializeKokkos.cpp @@ -19,36 +19,39 @@ namespace Tpetra { namespace Details { -void finalizeKokkosIfNeeded() { - if (!Kokkos::is_finalized()) { - Kokkos::finalize(); - } -} - void initializeKokkos() { - if (!Kokkos::is_initialized()) { - std::vector args = Teuchos::GlobalMPISession::getArgv(); - int narg = static_cast(args.size()); // must be nonconst - - std::vector args_c; - std::vector> args_; - for (auto const& x : args) { - args_.emplace_back(new char[x.size() + 1]); - char* ptr = args_.back().get(); - strcpy(ptr, x.c_str()); - args_c.push_back(ptr); + static const int initialized = []() { + TEUCHOS_TEST_FOR_EXCEPTION(Kokkos::is_finalized(), std::runtime_error, + "Tpetra::Details::initializeKokkos: Kokkos is already finalized"); + if (!Kokkos::is_initialized()) { + std::vector args = Teuchos::GlobalMPISession::getArgv(); + int narg = static_cast(args.size()); // must be nonconst + + std::vector args_c; + std::vector> args_; + for (auto const& x : args) { + args_.emplace_back(new char[x.size() + 1]); + char* ptr = args_.back().get(); + strcpy(ptr, x.c_str()); + args_c.push_back(ptr); + } + args_c.push_back(nullptr); + + Kokkos::initialize(narg, args_c.data()); + checkOldCudaLaunchBlocking(); + + std::atexit(Kokkos::finalize); } - args_c.push_back(nullptr); - Kokkos::initialize(narg, narg == 0 ? nullptr : args_c.data()); - checkOldCudaLaunchBlocking(); + // Add Kokkos calls to the TimeMonitor if the environment says so + Tpetra::Details::AddKokkosDeepCopyToTimeMonitor(); + Tpetra::Details::AddKokkosFenceToTimeMonitor(); + Tpetra::Details::AddKokkosFunctionsToTimeMonitor(); + return 1; + }(); - std::atexit(finalizeKokkosIfNeeded); - } - // Add Kokkos calls to the TimeMonitor if the environment says so - Tpetra::Details::AddKokkosDeepCopyToTimeMonitor(); - Tpetra::Details::AddKokkosFenceToTimeMonitor(); - Tpetra::Details::AddKokkosFunctionsToTimeMonitor(); + TEUCHOS_TEST_FOR_EXCEPTION(!Kokkos::is_initialized() || initialized != 1, std::runtime_error, + "Tpetra::Details::initializeKokkos: Initialization failed or Kokkos has already been finalized."); } } // namespace Details