[Mirror] 16.2 backports

Version.cmake

@@ -7,11 +7,11 @@
 SET(Trilinos_VERSION 16.2.0)
 SET(Trilinos_MAJOR_VERSION 16)
 SET(Trilinos_MAJOR_MINOR_VERSION 160200)
-SET(Trilinos_VERSION_STRING "16.2.0-dev")
-SET(Trilinos_ENABLE_DEVELOPMENT_MODE_DEFAULT ON) # Change to 'OFF' for a release
+SET(Trilinos_VERSION_STRING "16.2.0")
+SET(Trilinos_ENABLE_DEVELOPMENT_MODE_DEFAULT OFF) # Change to 'OFF' for a release
 
 # Used by testing scripts and should not be used elsewhere
-SET(Trilinos_REPOSITORY_BRANCH "develop" CACHE INTERNAL "")
+SET(Trilinos_REPOSITORY_BRANCH "trilinos-release-16-2-branch" CACHE INTERNAL "")
 SET(Trilinos_EXTRAREPOS_BRANCH "master" CACHE INTERNAL "")
 SET(Trilinos_TESTING_TRACK "" CACHE INTERNAL "")
 

cmake/ProjectCompilerPostConfig.cmake

@@ -32,6 +32,8 @@ macro(enable_errors errors)
     endforeach()
 endmacro()
 
+message(STATUS "Adding '-std=c99' to C compiler flags for Zoltan")
+set(Zoltan_C_FLAGS "-std=c99 ${Zoltan_C_FLAGS} ${CMAKE_C_FLAGS}")
 
 IF (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
   IF(WIN32)
@@ -205,7 +207,6 @@ function(filter_valid_warnings_as_errors warnings output)
     set(${output} ${valid_warnings} PARENT_SCOPE)
 endfunction()
 
-
 if("${Trilinos_WARNINGS_MODE}" STREQUAL "WARN")
     filter_valid_warnings("${upcoming_warnings}" upcoming_warnings)
     enable_warnings("${upcoming_warnings}")

packages/amesos2/src/Amesos2_CssMKL_def.hpp

@@ -140,6 +140,23 @@ namespace Amesos2 {
       Teuchos::TimeMonitor symbFactTimer( this->timers_.symFactTime_ );
 #endif
 
+      if (css_initialized_)
+      {
+        int_t phase = -1;         // release all internal solver memory
+        void *bdummy, *xdummy;
+        const MPI_Fint CssComm = CssComm_;
+        function_map::cluster_sparse_solver( pt_, const_cast<int_t*>(&maxfct_),
+                               const_cast<int_t*>(&mnum_), &mtype_, &phase, &n_,
+                               nzvals_view_.data(), rowptr_view_.data(),
+                               colind_view_.data(), perm_.getRawPtr(), &nrhs_, iparm_,
+                               const_cast<int_t*>(&msglvl_), &bdummy, &xdummy, &CssComm, &error );
+        css_initialized_ = false;
+        if (msglvl_ > 0 && error != 0 && this->matrixA_->getComm()->getRank() == 0) {
+          std::cout << " CssMKL::symbolicFactorization: clean-up failed with " << error << std::endl;
+        }
+      }
+
+      error = 0;
       int_t phase = 11; // Analysis
       void *bdummy, *xdummy;
       const MPI_Fint CssComm = CssComm_;
@@ -152,7 +169,11 @@ namespace Amesos2 {
     check_css_mkl_error(Amesos2::SYMBFACT, error);
     if (msglvl_ > 0 && this->matrixA_->getComm()->getRank() == 0) {
       std::cout << " CssMKL::symbolicFactorization done:" << std::endl;
+#ifdef HAVE_AMESOS2_TIMERS
       std::cout << " * Time : " << this->timers_.symFactTime_.totalElapsedTime() << std::endl;
+#else
+      std::cout << " * Time : not enabled" << std::endl;
+#endif
     }
 
     // Pardiso only lets you retrieve the total number of factor
@@ -190,7 +211,11 @@ namespace Amesos2 {
     check_css_mkl_error(Amesos2::NUMFACT, error);
     if (msglvl_ > 0 && this->matrixA_->getComm()->getRank() == 0) {
       std::cout << " CssMKL::numericFactorization done:" << std::endl;
-      std::cout << " Time : " << this->timers_.numFactTime_.totalElapsedTime() << std::endl;
+#ifdef HAVE_AMESOS2_TIMERS
+      std::cout << " * Time : " << this->timers_.numFactTime_.totalElapsedTime() << std::endl;
+#else
+      std::cout << " * Time : not enabled" << std::endl;
+#endif
     }
 
     return( 0 );
@@ -262,6 +287,15 @@ namespace Amesos2 {
           as<size_t>(ld_rhs),
           Teuchos::ptrInArg(*css_rowmap_));
     }
+    if (msglvl_ > 0 && this->matrixA_->getComm()->getRank() == 0) {
+      std::cout << " CssMKL::solve done:" << std::endl;
+#ifdef HAVE_AMESOS2_TIMERS
+      std::cout << " * Time : " << this->timers_.vecRedistTime_.totalElapsedTime()
+                << " + " << this->timers_.solveTime_.totalElapsedTime() << std::endl;
+#else
+      std::cout << " * Time : not enabled" << std::endl;
+#endif
+    }
 
     return( 0 );
   }

packages/amesos2/src/Amesos2_KLU2_TypeMap.hpp

@@ -61,7 +61,6 @@ struct TypeMap<KLU2,double>
 };
 
 #ifdef HAVE_TEUCHOS_COMPLEX
-
 template <>
 struct TypeMap<KLU2,std::complex<float> >
 {
@@ -89,12 +88,9 @@ struct TypeMap<KLU2,Kokkos::complex<double> >
   typedef std::complex<double> dtype;
   typedef Kokkos::complex<double> type;
 };
-
 #endif  // HAVE_TEUCHOS_COMPLEX
-
 /* \endcond KLU2_type_specializations */
 
-
 } // end namespace Amesos2
 
 #endif  // AMESOS2_SUPERLU_TYPEMAP_HPP

packages/amesos2/src/Amesos2_KLU2_decl.hpp

@@ -230,6 +230,8 @@ class KLU2 : public SolverCore<Amesos2::KLU2, Matrix, Vector>
 
   bool is_contiguous_;
   bool use_gather_;
+
+  int debug_level_;
 };                              // End class KLU2
 
 

packages/amesos2/src/Amesos2_KLU2_def.hpp

@@ -37,6 +37,7 @@ KLU2<Matrix,Vector>::KLU2(
   , transFlag_(0)
   , is_contiguous_(true)
   , use_gather_(true)
+  , debug_level_(0)
 {
   ::KLU2::klu_defaults<klu2_dtype, local_ordinal_type> (&(data_.common_)) ;
   data_.symbolic_ = NULL;
@@ -171,6 +172,19 @@ KLU2<Matrix,Vector>::numericFactorization_impl()
       // is the only error/throw we currently have a unit test for.
       if(data_.numeric_ == nullptr) {
         info = 1;
+        if(debug_level_ > 0) {
+          std::cout << " ** Amesos2::KLU2::numericFactorization failed with status = ";
+          if(data_.common_.status == KLU_OK)
+            std::cout << "KLU_OK **\n";
+          else if (data_.common_.status == KLU_SINGULAR)
+            std::cout << "KLU_SINGULAR **\n";
+          else if (data_.common_.status == KLU_OUT_OF_MEMORY)
+            std::cout << "KLU_OUT_OF_MEMORY **\n";
+          else if (data_.common_.status == KLU_INVALID)
+            std::cout << "KLU_INVALID **\n";
+          else if (data_.common_.status == KLU_TOO_LARGE) 
+            std::cout << "KLU_TOO_LARGE **\n";
+        }
       }
 
       // This is set after numeric factorization complete as pivoting can be used;
@@ -202,6 +216,17 @@ KLU2<Matrix,Vector>::solve_impl(
 
   const global_size_type ld_rhs = this->root_ ? X->getGlobalLength() : 0;
   const size_t nrhs = X->getGlobalNumVectors();
+  if (debug_level_ > 0) {
+    if (this->root_) std::cout << "\n == Amesos2_KLU2::solve_impl ==" << std::endl;
+    if (debug_level_ == 1) {
+      B->description();
+    } else {
+      Teuchos::RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
+      B->getMap()->describe(*fancy, Teuchos::VERB_EXTREME);
+      std::cout << std::endl;
+      B->describe(*fancy, Teuchos::VERB_EXTREME);
+    }
+  }
 
   bool bDidAssignX;
   bool bDidAssignB;
@@ -374,6 +399,16 @@ KLU2<Matrix,Vector>::solve_impl(
           this->rowIndexBase_);
     }
   }
+  if (debug_level_ > 0) {
+    if (debug_level_ == 1) {
+      X->description();
+    } else {
+      Teuchos::RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
+      X->getMap()->describe(*fancy, Teuchos::VERB_EXTREME);
+      std::cout << std::endl;
+      X->describe(*fancy, Teuchos::VERB_EXTREME);
+    }
+  }
   return(ierr);
 }
 
@@ -414,6 +449,10 @@ KLU2<Matrix,Vector>::setParameters_impl(const Teuchos::RCP<Teuchos::ParameterLis
   if( parameterList->isParameter("UseCustomGather") ){
     use_gather_ = parameterList->get<bool>("UseCustomGather");
   }
+
+  if( parameterList->isParameter("DebugLevel") ){
+    debug_level_ = parameterList->get<int>("DebugLevel");
+  }
 }
 
 
@@ -435,6 +474,7 @@ KLU2<Matrix,Vector>::getValidParameters_impl() const
     pl->set("Equil", true, "Whether to equilibrate the system before solve, does nothing now");
     pl->set("IsContiguous", true, "Whether GIDs contiguous");
     pl->set("UseCustomGather", true, "Whether to use new matrix-gather routine");
+    pl->set("DebugLevel", 0, "Debug message level (0 for no message, and >0 for more message");
 
     setStringToIntegralParameter<int>("Trans", "NOTRANS",
                                       "Solve for the transpose system or not",
@@ -461,6 +501,11 @@ KLU2<Matrix,Vector>::loadA_impl(EPhase current_phase)
 #endif
 
   if(current_phase == SOLVE)return(false);
+  if (debug_level_ > 0 && current_phase == NUMFACT) {
+    if (this->root_) std::cout << "\n == Amesos2_KLU2::loadA_impl(NumFact) ==\n" << std::endl;
+    Teuchos::RCP<Teuchos::FancyOStream> fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
+    this->matrixA_->describe(*fancy, Teuchos::VERB_EXTREME);
+  }
 
   if ( single_proc_optimization() ) {
     // Do nothing in this case - Crs raw pointers will be used

packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_decl.hpp

@@ -126,8 +126,13 @@ namespace Amesos2 {
                              host_ordinal_type_array &recvCounts, host_ordinal_type_array &recvDispls,
                              host_ordinal_type_array &transpose_map, host_scalar_type_array &nzvals_t,
                              bool column_major, EPhase current_phase) const;
-  };
 
+    //! Print a description of this adapter to the given output stream
+    void
+    describe (Teuchos::FancyOStream& os,
+              const Teuchos::EVerbosityLevel verbLevel =
+              Teuchos::Describable::verbLevel_default) const;
+  };
 } // end namespace Amesos2
 
 #endif  // AMESOS2_KOKKOSCRSMATRIX_MATRIXADAPTER_DECL_HPP

packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_def.hpp

@@ -201,6 +201,20 @@ namespace Amesos2 {
     //TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, "KokkosCrsMatrixAdapter has not been implemented gather_impl.");
     return -1;
   }
+
+
+  template <typename Scalar, typename LocalOrdinal, typename ExecutionSpace>
+  void
+  ConcreteMatrixAdapter<
+    KokkosSparse::CrsMatrix<Scalar,LocalOrdinal,ExecutionSpace>
+    >::describe (Teuchos::FancyOStream& os,
+                   const Teuchos::EVerbosityLevel verbLevel) const
+  {
+    size_t m = this->mat_->numRows();
+    size_t n = this->mat_->numCols();
+    os << " KokkosSparse::CrsMatrix(" << std::to_string(m) << " x " << std::to_string(n) << ")";
+    os << " of type " << std::string(typeid(Scalar).name()) << std::endl;
+  }
 } // end namespace Amesos2
 
 #endif  // AMESOS2_KOKKOS_CRSMATRIX_MATRIXADAPTER_DEF_HPP

packages/amesos2/src/Amesos2_KokkosMultiVecAdapter_def.hpp

@@ -123,8 +123,11 @@ namespace Amesos2 {
     Kokkos::View<Scalar**, Kokkos::LayoutLeft, ExecutionSpace> >::description() const
   {
     std::ostringstream oss;
+    size_t m = mv_->extent(0);
+    size_t n = mv_->extent(1);
     oss << "Amesos2 adapter wrapping: ";
-    oss << mv_->description();
+    oss << " Kokkos::View(" << std::to_string(m) << " x " << std::to_string(n) << ")";
+    oss << " of type " << std::string(typeid(Scalar).name());
     return oss.str();
   }
 
@@ -135,7 +138,10 @@ namespace Amesos2 {
     Kokkos::View<Scalar**, Kokkos::LayoutLeft, ExecutionSpace> >::describe (Teuchos::FancyOStream& os,
                                    const Teuchos::EVerbosityLevel verbLevel) const
   {
-    mv_->describe (os, verbLevel);
+    size_t m = mv_->extent(0);
+    size_t n = mv_->extent(1);
+    os << " Kokkos::View(" << std::to_string(m) << " x " << std::to_string(n) << ")";
+    os << " of type " << std::string(typeid(Scalar).name()) << std::endl;
   }
 
 } // end namespace Amesos2

packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp

@@ -266,9 +266,6 @@ namespace Amesos2 {
     /// 1-based indexing, but our interface must use 0-based indexing
     int_t iparm_[64];
 
-    /// The messaging level.  Set to 1 if you wish for Pardiso MKL to print statistical info
-    static const int_t msglvl_;
-
     // We will deal with 1 factor at a time
     static const int_t maxfct_;
     static const int_t mnum_;
@@ -278,8 +275,12 @@ namespace Amesos2 {
     = Meta::or_<std::is_same_v<solver_scalar_type, PMKL::_MKL_Complex8>,
                 std::is_same_v<solver_scalar_type, PMKL::_DOUBLE_COMPLEX_t>>::value;
 
+    bool pardiso_initialized_;
     bool is_contiguous_;
 
+    /// The messaging level.  Set to 1 if you wish for Pardiso MKL to print statistical info
+    int_t msglvl_;
+    int debug_level_;
 };                              // End class PardisoMKL