From 338847390a8cb57a3a6f329a1ddeecc8dabe8b19 Mon Sep 17 00:00:00 2001 From: "Samuel E. Browne" Date: Mon, 24 Nov 2025 14:09:35 -0700 Subject: [PATCH 01/11] Update version(s) for 16.2.0 release Signed-off-by: Samuel E. Browne --- Version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Version.cmake b/Version.cmake index 796944707a78..1df452c42b8c 100644 --- a/Version.cmake +++ b/Version.cmake @@ -7,11 +7,11 @@ SET(Trilinos_VERSION 16.2.0) SET(Trilinos_MAJOR_VERSION 16) SET(Trilinos_MAJOR_MINOR_VERSION 160200) -SET(Trilinos_VERSION_STRING "16.2.0-dev") -SET(Trilinos_ENABLE_DEVELOPMENT_MODE_DEFAULT ON) # Change to 'OFF' for a release +SET(Trilinos_VERSION_STRING "16.2.0") +SET(Trilinos_ENABLE_DEVELOPMENT_MODE_DEFAULT OFF) # Change to 'OFF' for a release # Used by testing scripts and should not be used elsewhere -SET(Trilinos_REPOSITORY_BRANCH "develop" CACHE INTERNAL "") +SET(Trilinos_REPOSITORY_BRANCH "trilinos-release-16-2-branch" CACHE INTERNAL "") SET(Trilinos_EXTRAREPOS_BRANCH "master" CACHE INTERNAL "") SET(Trilinos_TESTING_TRACK "" CACHE INTERNAL "") From 138d115370ec222f693d6a29196c6b4b248075be Mon Sep 17 00:00:00 2001 From: "Samuel E. Browne" Date: Tue, 25 Nov 2025 07:52:35 -0700 Subject: [PATCH 02/11] Add `-std=c99` to C compiler flags Zoltan_ch_simple_zoltan_serial fails without this flag on GCC14 _without_ MPI enabled. It was getting added via the strong C warnings flags, so with DEVELOPMENT_MODE disabled it doesn't get added. Force it on here. Signed-off-by: Samuel E. Browne --- cmake/ProjectCompilerPostConfig.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/ProjectCompilerPostConfig.cmake b/cmake/ProjectCompilerPostConfig.cmake index 855690e0c620..1531518fa3b0 100644 --- a/cmake/ProjectCompilerPostConfig.cmake +++ b/cmake/ProjectCompilerPostConfig.cmake @@ -32,6 +32,8 @@ macro(enable_errors errors) endforeach() endmacro() +message(STATUS "Adding '-std=c99' to C compiler flags for Zoltan") +set(Zoltan_C_FLAGS "-std=c99 ${Zoltan_C_FLAGS} ${CMAKE_C_FLAGS}") IF (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") IF(WIN32) @@ -205,7 +207,6 @@ function(filter_valid_warnings_as_errors warnings output) set(${output} ${valid_warnings} PARENT_SCOPE) endfunction() - if("${Trilinos_WARNINGS_MODE}" STREQUAL "WARN") filter_valid_warnings("${upcoming_warnings}" upcoming_warnings) enable_warnings("${upcoming_warnings}") From 85c4054105ac7ef10f221dc127e508fa9e79baa2 Mon Sep 17 00:00:00 2001 From: Eric Phipps Date: Mon, 15 Dec 2025 12:15:45 -0700 Subject: [PATCH 03/11] Stokhos: Add Tpetra utilities to replace now missing Epetra utilities This adds new Tpetra replacements for some sparse 3-tensor utilities that have now disappeared from the Epetra removal. Signed-off-by: Eric Phipps --- packages/stokhos/src/CMakeLists.txt | 7 + .../Stokhos_Tpetra_Sparse3TensorUtilities.hpp | 105 ++++++++++++ packages/stokhos/test/UnitTest/CMakeLists.txt | 12 ++ ...khos_Tpetra_Sparse3TensorUtilsUnitTest.cpp | 157 ++++++++++++++++++ 4 files changed, 281 insertions(+) create mode 100644 packages/stokhos/src/tpetra/Stokhos_Tpetra_Sparse3TensorUtilities.hpp create mode 100644 packages/stokhos/test/UnitTest/Stokhos_Tpetra_Sparse3TensorUtilsUnitTest.cpp diff --git a/packages/stokhos/src/CMakeLists.txt b/packages/stokhos/src/CMakeLists.txt index dbfb8cd98b10..3aabd81b4c40 100644 --- a/packages/stokhos/src/CMakeLists.txt +++ b/packages/stokhos/src/CMakeLists.txt @@ -516,6 +516,13 @@ IF (Stokhos_ENABLE_EpetraExt) ) ENDIF() +IF (Stokhos_ENABLE_Tpetra) + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/tpetra) + SET(HEADERS ${HEADERS} + tpetra/Stokhos_Tpetra_Sparse3TensorUtilities.hpp + ) +ENDIF() + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/kokkos) TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/kokkos/kl) SET(HEADERS ${HEADERS} diff --git a/packages/stokhos/src/tpetra/Stokhos_Tpetra_Sparse3TensorUtilities.hpp b/packages/stokhos/src/tpetra/Stokhos_Tpetra_Sparse3TensorUtilities.hpp new file mode 100644 index 000000000000..5445b1e6cacf --- /dev/null +++ b/packages/stokhos/src/tpetra/Stokhos_Tpetra_Sparse3TensorUtilities.hpp @@ -0,0 +1,105 @@ +// @HEADER +// ***************************************************************************** +// Stokhos Package +// +// Copyright 2009 NTESS and the Stokhos contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#ifndef STOKHOS_TPETRA_SPARSE_3_TENSOR_UTILITIES_HPP +#define STOKHOS_TPETRA_SPARSE_3_TENSOR_UTILITIES_HPP + +#include "Stokhos_Sparse3Tensor.hpp" +#include "Tpetra_Map.hpp" +#include "Tpetra_CrsGraph.hpp" +#include "MatrixMarket_Tpetra.hpp" + +namespace Stokhos { + + //! Build an Tpetra::CrsGraph from a sparse 3 tensor + /*! + * Builds a sparse graph from a sparse 3 tensor by summing over the third + * index. This graph then represents the sparsity pattern of the stochastic + * part of the block stochastic Galerkin operator. Redistributing the graph + * should then provide a suitable parallel distribution for block + * stochastic Galerkin linear solves. + */ + template + Teuchos::RCP< Tpetra::CrsGraph > + sparse3Tensor2TpetraCrsGraph( + const Stokhos::OrthogPolyBasis& basis, + const Stokhos::Sparse3Tensor& Cijk, + const Teuchos::RCP >& comm) + { + typedef Stokhos::Sparse3Tensor Cijk_type; + typedef Tpetra::Map Map; + typedef Tpetra::CrsGraph Graph; + using Teuchos::RCP; + using Teuchos::arrayView; + + // Number of stochastic rows + StokhosOrdinal num_rows = basis.size(); + + // Compute max num entries -- note this is a substantial overestimate in most cases because it doesn't + // take into account overlap in j entries for each k. Also, while Cijk is symmetric with respect to i and + // j, it is not necessarily symmetric with respect to k. So this is the best we can do without adding + // and i -> j -> k traversal to the Cijk tensor. + size_t max_num_entry = 0; + for (typename Cijk_type::i_iterator i_it=Cijk.i_begin(); i_it!=Cijk.i_end(); ++i_it) { + size_t num_entry = 0; + for (typename Cijk_type::ik_iterator k_it = Cijk.k_begin(i_it); k_it != Cijk.k_end(i_it); ++k_it) { + for (typename Cijk_type::ikj_iterator j_it = Cijk.j_begin(k_it); j_it != Cijk.j_end(k_it); ++j_it) { + ++num_entry; + } + max_num_entry = (num_entry > max_num_entry) ? num_entry : max_num_entry; + } + } + + // Replicated local map + RCP map = + Tpetra::createLocalMapWithNode(num_rows, comm); + + // Graph to be created + RCP graph = Tpetra::createCrsGraph(map, max_num_entry); + + // Loop over Cijk entries including a non-zero in the graph at + // indices (i,j) if there is any k for which Cijk is non-zero + for (typename Cijk_type::i_iterator i_it=Cijk.i_begin(); i_it!=Cijk.i_end(); ++i_it) { + TpetraGlobalOrdinal i = index(i_it); + for (typename Cijk_type::ik_iterator k_it = Cijk.k_begin(i_it); k_it != Cijk.k_end(i_it); ++k_it) { + for (typename Cijk_type::ikj_iterator j_it = Cijk.j_begin(k_it); j_it != Cijk.j_end(k_it); ++j_it) { + TpetraGlobalOrdinal j = index(j_it); + graph->insertGlobalIndices(i, arrayView(&j, 1)); + } + } + } + + // Sort, remove redundencies, transform to local, ... + graph->fillComplete(); + + return graph; + } + + template + void + sparse3Tensor2TpetraMatrixMarket( + const Stokhos::OrthogPolyBasis& basis, + const Stokhos::Sparse3Tensor& Cijk, + const Teuchos::RCP >& comm, + const std::string& file) + { + typedef Tpetra::CrsGraph Graph; + typedef Tpetra::CrsMatrix Matrix; + typedef Tpetra::MatrixMarket::Writer Writer; + + Teuchos::RCP graph = + Stokhos::sparse3Tensor2TpetraCrsGraph( + basis, Cijk, comm); + Writer::writeSparseGraphFile(file, *graph); + } +} // namespace Stokhos + +#endif // STOKHOS_TPETRA_SPARSE_3_TENSOR_UTILITIES_HPP diff --git a/packages/stokhos/test/UnitTest/CMakeLists.txt b/packages/stokhos/test/UnitTest/CMakeLists.txt index 9942590015f3..319b489bf239 100644 --- a/packages/stokhos/test/UnitTest/CMakeLists.txt +++ b/packages/stokhos/test/UnitTest/CMakeLists.txt @@ -323,6 +323,18 @@ IF (Stokhos_ENABLE_EpetraExt) ENDIF() +IF (Stokhos_ENABLE_Tpetra) + + TRIBITS_ADD_EXECUTABLE_AND_TEST( + TpetraSparse3TensorUtilsUnitTest + SOURCES Stokhos_Tpetra_Sparse3TensorUtilsUnitTest.cpp + COMM serial mpi + STANDARD_PASS_OUTPUT + NUM_MPI_PROCS 1 + ) + +ENDIF() + IF(Stokhos_ENABLE_Sacado) TRIBITS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${PACKAGE_SOURCE_DIR}/../sacado/test/utils) diff --git a/packages/stokhos/test/UnitTest/Stokhos_Tpetra_Sparse3TensorUtilsUnitTest.cpp b/packages/stokhos/test/UnitTest/Stokhos_Tpetra_Sparse3TensorUtilsUnitTest.cpp new file mode 100644 index 000000000000..0f4e8600c866 --- /dev/null +++ b/packages/stokhos/test/UnitTest/Stokhos_Tpetra_Sparse3TensorUtilsUnitTest.cpp @@ -0,0 +1,157 @@ +// @HEADER +// ***************************************************************************** +// Stokhos Package +// +// Copyright 2009 NTESS and the Stokhos contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include "Teuchos_UnitTestHarness.hpp" +#include "Teuchos_TestingHelpers.hpp" +#include "Teuchos_UnitTestRepository.hpp" +#include "Teuchos_GlobalMPISession.hpp" + +#include "Stokhos.hpp" +#include "Stokhos_Tpetra_Sparse3TensorUtilities.hpp" +#include "Stokhos_UnitTestHelpers.hpp" + +namespace TpetraSparse3TensorUnitTest { + + typedef Stokhos::Sparse3Tensor Cijk_type; + + // Common setup for unit tests + template + struct UnitTestSetup { + ValueType rtol, atol, sparse_tol; + OrdinalType d, p, sz; + Teuchos::Array< Teuchos::RCP > > bases; + Teuchos::RCP > basis; + Teuchos::RCP Cijk; + + UnitTestSetup(): d(3), p(4), bases(d) { + rtol = 1e-14; + atol = 1e-14; + sparse_tol = 1e-14; + + // Create product basis + for (OrdinalType i=0; i(p)); + basis = + Teuchos::rcp(new Stokhos::CompletePolynomialBasis(bases, sparse_tol)); + sz = basis->size(); + + // Triple product tensor + Cijk = basis->computeTripleProductTensor(); + } + + }; + + UnitTestSetup setup; + + TEUCHOS_UNIT_TEST( Stokhos_TpetraSparse3Tensor, Graph ) { + using TpetraLocalOrdinal = Tpetra::Details::DefaultTypes::local_ordinal_type; + using TpetraGlobalOrdinal = Tpetra::Details::DefaultTypes::global_ordinal_type; + using TpetraNode = Tpetra::Details::DefaultTypes::node_type; + using TpetraGraph = Tpetra::CrsGraph; + using TpetraComm = Teuchos::Comm; + using host_view_type = TpetraGraph::nonconst_local_inds_host_view_type; + + // Create MPI comm + Teuchos::RCP comm = Tpetra::getDefaultComm(); + + // Create CrsGraph from Cijk tensor + Teuchos::RCP graph = + Stokhos::sparse3Tensor2TpetraCrsGraph( + *setup.basis, *setup.Cijk, comm); + + // Check all entries in Cijk are in the graph + success = true; + for (Cijk_type::k_iterator k_it=setup.Cijk->k_begin(); k_it!=setup.Cijk->k_end(); ++k_it) { + for (Cijk_type::kj_iterator j_it = setup.Cijk->j_begin(k_it); j_it != setup.Cijk->j_end(k_it); ++j_it) { + TpetraLocalOrdinal j = Stokhos::index(j_it); + for (Cijk_type::kji_iterator i_it = setup.Cijk->i_begin(j_it); i_it != setup.Cijk->i_end(j_it); ++i_it) { + TpetraLocalOrdinal i = Stokhos::index(i_it); + size_t num_col_inds = graph->getNumAllocatedEntriesInLocalRow(i); + host_view_type col_inds("col_inds", num_col_inds); + graph->getLocalRowCopy(i, col_inds, num_col_inds); + bool my_success = false; + for (size_t l=0; l; + using TpetraMatrix = Tpetra::CrsMatrix; + using TpetraReader = Tpetra::MatrixMarket::Reader; + using TpetraComm = Teuchos::Comm; + using host_view_type = TpetraGraph::nonconst_local_inds_host_view_type; + + // Create MPI comm + Teuchos::RCP comm = Tpetra::getDefaultComm(); + + // Write Cijk to matrix market file + const std::string mm_file = "Cijk.mm"; + Stokhos::sparse3Tensor2TpetraMatrixMarket( + *setup.basis, *setup.Cijk, comm, mm_file); + + // Read graph from file + Teuchos::RCP graph = TpetraReader::readSparseGraphFile(mm_file, comm); + + // Check all entries in Cijk are in the graph + success = true; + for (Cijk_type::k_iterator k_it=setup.Cijk->k_begin(); k_it!=setup.Cijk->k_end(); ++k_it) { + for (Cijk_type::kj_iterator j_it = setup.Cijk->j_begin(k_it); j_it != setup.Cijk->j_end(k_it); ++j_it) { + TpetraLocalOrdinal j = Stokhos::index(j_it); + for (Cijk_type::kji_iterator i_it = setup.Cijk->i_begin(j_it); i_it != setup.Cijk->i_end(j_it); ++i_it) { + TpetraLocalOrdinal i = Stokhos::index(i_it); + size_t num_col_inds = graph->getNumAllocatedEntriesInLocalRow(i); + host_view_type col_inds("col_inds", num_col_inds); + graph->getLocalRowCopy(i, col_inds, num_col_inds); + bool my_success = false; + for (size_t l=0; l Date: Tue, 16 Dec 2025 15:11:22 -0700 Subject: [PATCH 04/11] Amesos2: Debug info option for KLU2 & PardisoMKL (#14808) * Amesos2: Pardiso with debug statements Signed-off-by: iyamazaki * Amesos2: print X after solve Signed-off-by: iyamazaki * Amesos2:KLU2 print A, b, & x for debuging Signed-off-by: iyamazaki * Amesos2: print map for Pardiso/KLU2 from solve_impl Signed-off-by: iyamazaki * Amesos2: turn of debug message for default KLU2 & Pardiso and remove some debugging printf Signed-off-by: iyamazaki * Amesos2: switching to std::cout Signed-off-by: iyamazaki * Amesos2: comments about debug and message levels Signed-off-by: iyamazaki --------- Signed-off-by: iyamazaki --- packages/amesos2/src/Amesos2_KLU2_TypeMap.hpp | 4 -- packages/amesos2/src/Amesos2_KLU2_decl.hpp | 2 + packages/amesos2/src/Amesos2_KLU2_def.hpp | 45 ++++++++++++++++++ ...os2_KokkosCrsMatrix_MatrixAdapter_decl.hpp | 7 ++- ...sos2_KokkosCrsMatrix_MatrixAdapter_def.hpp | 14 ++++++ .../src/Amesos2_KokkosMultiVecAdapter_def.hpp | 10 +++- .../amesos2/src/Amesos2_PardisoMKL_decl.hpp | 6 +-- .../amesos2/src/Amesos2_PardisoMKL_def.hpp | 46 +++++++++++++++---- 8 files changed, 115 insertions(+), 19 deletions(-) diff --git a/packages/amesos2/src/Amesos2_KLU2_TypeMap.hpp b/packages/amesos2/src/Amesos2_KLU2_TypeMap.hpp index e701c865932d..32da411bc2c6 100644 --- a/packages/amesos2/src/Amesos2_KLU2_TypeMap.hpp +++ b/packages/amesos2/src/Amesos2_KLU2_TypeMap.hpp @@ -61,7 +61,6 @@ struct TypeMap }; #ifdef HAVE_TEUCHOS_COMPLEX - template <> struct TypeMap > { @@ -89,12 +88,9 @@ struct TypeMap > typedef std::complex dtype; typedef Kokkos::complex type; }; - #endif // HAVE_TEUCHOS_COMPLEX - /* \endcond KLU2_type_specializations */ - } // end namespace Amesos2 #endif // AMESOS2_SUPERLU_TYPEMAP_HPP diff --git a/packages/amesos2/src/Amesos2_KLU2_decl.hpp b/packages/amesos2/src/Amesos2_KLU2_decl.hpp index e456c6e78ccc..c11bd0f1ce0e 100644 --- a/packages/amesos2/src/Amesos2_KLU2_decl.hpp +++ b/packages/amesos2/src/Amesos2_KLU2_decl.hpp @@ -230,6 +230,8 @@ class KLU2 : public SolverCore bool is_contiguous_; bool use_gather_; + + int debug_level_; }; // End class KLU2 diff --git a/packages/amesos2/src/Amesos2_KLU2_def.hpp b/packages/amesos2/src/Amesos2_KLU2_def.hpp index 3ee4aa4c1705..8642296b86d9 100644 --- a/packages/amesos2/src/Amesos2_KLU2_def.hpp +++ b/packages/amesos2/src/Amesos2_KLU2_def.hpp @@ -37,6 +37,7 @@ KLU2::KLU2( , transFlag_(0) , is_contiguous_(true) , use_gather_(true) + , debug_level_(0) { ::KLU2::klu_defaults (&(data_.common_)) ; data_.symbolic_ = NULL; @@ -171,6 +172,19 @@ KLU2::numericFactorization_impl() // is the only error/throw we currently have a unit test for. if(data_.numeric_ == nullptr) { info = 1; + if(debug_level_ > 0) { + std::cout << " ** Amesos2::KLU2::numericFactorization failed with status = "; + if(data_.common_.status == KLU_OK) + std::cout << "KLU_OK **\n"; + else if (data_.common_.status == KLU_SINGULAR) + std::cout << "KLU_SINGULAR **\n"; + else if (data_.common_.status == KLU_OUT_OF_MEMORY) + std::cout << "KLU_OUT_OF_MEMORY **\n"; + else if (data_.common_.status == KLU_INVALID) + std::cout << "KLU_INVALID **\n"; + else if (data_.common_.status == KLU_TOO_LARGE) + std::cout << "KLU_TOO_LARGE **\n"; + } } // This is set after numeric factorization complete as pivoting can be used; @@ -202,6 +216,17 @@ KLU2::solve_impl( const global_size_type ld_rhs = this->root_ ? X->getGlobalLength() : 0; const size_t nrhs = X->getGlobalNumVectors(); + if (debug_level_ > 0) { + if (this->root_) std::cout << "\n == Amesos2_KLU2::solve_impl ==" << std::endl; + if (debug_level_ == 1) { + B->description(); + } else { + Teuchos::RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + B->getMap()->describe(*fancy, Teuchos::VERB_EXTREME); + std::cout << std::endl; + B->describe(*fancy, Teuchos::VERB_EXTREME); + } + } bool bDidAssignX; bool bDidAssignB; @@ -374,6 +399,16 @@ KLU2::solve_impl( this->rowIndexBase_); } } + if (debug_level_ > 0) { + if (debug_level_ == 1) { + X->description(); + } else { + Teuchos::RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + X->getMap()->describe(*fancy, Teuchos::VERB_EXTREME); + std::cout << std::endl; + X->describe(*fancy, Teuchos::VERB_EXTREME); + } + } return(ierr); } @@ -414,6 +449,10 @@ KLU2::setParameters_impl(const Teuchos::RCPisParameter("UseCustomGather") ){ use_gather_ = parameterList->get("UseCustomGather"); } + + if( parameterList->isParameter("DebugLevel") ){ + debug_level_ = parameterList->get("DebugLevel"); + } } @@ -435,6 +474,7 @@ KLU2::getValidParameters_impl() const pl->set("Equil", true, "Whether to equilibrate the system before solve, does nothing now"); pl->set("IsContiguous", true, "Whether GIDs contiguous"); pl->set("UseCustomGather", true, "Whether to use new matrix-gather routine"); + pl->set("DebugLevel", 0, "Debug message level (0 for no message, and >0 for more message"); setStringToIntegralParameter("Trans", "NOTRANS", "Solve for the transpose system or not", @@ -461,6 +501,11 @@ KLU2::loadA_impl(EPhase current_phase) #endif if(current_phase == SOLVE)return(false); + if (debug_level_ > 0 && current_phase == NUMFACT) { + if (this->root_) std::cout << "\n == Amesos2_KLU2::loadA_impl(NumFact) ==\n" << std::endl; + Teuchos::RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + this->matrixA_->describe(*fancy, Teuchos::VERB_EXTREME); + } if ( single_proc_optimization() ) { // Do nothing in this case - Crs raw pointers will be used diff --git a/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_decl.hpp b/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_decl.hpp index f93470f81b2c..e6db68900451 100644 --- a/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_decl.hpp +++ b/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_decl.hpp @@ -126,8 +126,13 @@ namespace Amesos2 { host_ordinal_type_array &recvCounts, host_ordinal_type_array &recvDispls, host_ordinal_type_array &transpose_map, host_scalar_type_array &nzvals_t, bool column_major, EPhase current_phase) const; - }; + //! Print a description of this adapter to the given output stream + void + describe (Teuchos::FancyOStream& os, + const Teuchos::EVerbosityLevel verbLevel = + Teuchos::Describable::verbLevel_default) const; + }; } // end namespace Amesos2 #endif // AMESOS2_KOKKOSCRSMATRIX_MATRIXADAPTER_DECL_HPP diff --git a/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_def.hpp b/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_def.hpp index 9a6f9fd1ea70..3089179a9fe4 100644 --- a/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_def.hpp +++ b/packages/amesos2/src/Amesos2_KokkosCrsMatrix_MatrixAdapter_def.hpp @@ -201,6 +201,20 @@ namespace Amesos2 { //TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, "KokkosCrsMatrixAdapter has not been implemented gather_impl."); return -1; } + + + template + void + ConcreteMatrixAdapter< + KokkosSparse::CrsMatrix + >::describe (Teuchos::FancyOStream& os, + const Teuchos::EVerbosityLevel verbLevel) const + { + size_t m = this->mat_->numRows(); + size_t n = this->mat_->numCols(); + os << " KokkosSparse::CrsMatrix(" << std::to_string(m) << " x " << std::to_string(n) << ")"; + os << " of type " << std::string(typeid(Scalar).name()) << std::endl; + } } // end namespace Amesos2 #endif // AMESOS2_KOKKOS_CRSMATRIX_MATRIXADAPTER_DEF_HPP diff --git a/packages/amesos2/src/Amesos2_KokkosMultiVecAdapter_def.hpp b/packages/amesos2/src/Amesos2_KokkosMultiVecAdapter_def.hpp index 30e14124a61a..9c9d2a5492bf 100644 --- a/packages/amesos2/src/Amesos2_KokkosMultiVecAdapter_def.hpp +++ b/packages/amesos2/src/Amesos2_KokkosMultiVecAdapter_def.hpp @@ -123,8 +123,11 @@ namespace Amesos2 { Kokkos::View >::description() const { std::ostringstream oss; + size_t m = mv_->extent(0); + size_t n = mv_->extent(1); oss << "Amesos2 adapter wrapping: "; - oss << mv_->description(); + oss << " Kokkos::View(" << std::to_string(m) << " x " << std::to_string(n) << ")"; + oss << " of type " << std::string(typeid(Scalar).name()); return oss.str(); } @@ -135,7 +138,10 @@ namespace Amesos2 { Kokkos::View >::describe (Teuchos::FancyOStream& os, const Teuchos::EVerbosityLevel verbLevel) const { - mv_->describe (os, verbLevel); + size_t m = mv_->extent(0); + size_t n = mv_->extent(1); + os << " Kokkos::View(" << std::to_string(m) << " x " << std::to_string(n) << ")"; + os << " of type " << std::string(typeid(Scalar).name()) << std::endl; } } // end namespace Amesos2 diff --git a/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp b/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp index e9539efba816..067a9b4ec7ab 100644 --- a/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp +++ b/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp @@ -266,9 +266,6 @@ namespace Amesos2 { /// 1-based indexing, but our interface must use 0-based indexing int_t iparm_[64]; - /// The messaging level. Set to 1 if you wish for Pardiso MKL to print statistical info - static const int_t msglvl_; - // We will deal with 1 factor at a time static const int_t maxfct_; static const int_t mnum_; @@ -280,6 +277,9 @@ namespace Amesos2 { bool is_contiguous_; + /// The messaging level. Set to 1 if you wish for Pardiso MKL to print statistical info + int_t msglvl_; + int debug_level_; }; // End class PardisoMKL diff --git a/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp b/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp index 9aa0a0c853a0..e2a79bad2edb 100644 --- a/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp +++ b/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp @@ -45,6 +45,8 @@ namespace Amesos2 { , perm_(this->globalNumRows_) , nrhs_(0) , is_contiguous_(true) + , msglvl_(0) + , debug_level_(0) { // set the default matrix type set_pardiso_mkl_matrix_type(); @@ -150,7 +152,6 @@ namespace Amesos2 { int_t phase = 22; void *bdummy, *xdummy; - function_map::pardiso( pt_, const_cast(&maxfct_), const_cast(&mnum_), &mtype_, &phase, &n_, nzvals_view_.data(), rowptr_view_.data(), @@ -176,6 +177,17 @@ namespace Amesos2 { // Get B data const global_size_type ld_rhs = this->root_ ? X->getGlobalLength() : 0; nrhs_ = as(X->getGlobalNumVectors()); + if (debug_level_ > 0) { + if (this->root_) std::cout << "\n == Amesos2_PardisoMKL::solve_impl ==" << std::endl; + if (debug_level_ == 1) { + B->description(); + } else { + Teuchos::RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + B->getMap()->describe(*fancy, Teuchos::VERB_EXTREME); + std::cout << std::endl; + B->describe(*fancy, Teuchos::VERB_EXTREME); + } + } const size_t val_store_size = as(ld_rhs * nrhs_); xvals_.resize(val_store_size); @@ -201,7 +213,6 @@ namespace Amesos2 { #endif const int_t phase = 33; - function_map::pardiso( pt_, const_cast(&maxfct_), const_cast(&mnum_), @@ -232,7 +243,17 @@ namespace Amesos2 { solver_scalar_type>::do_put(X, xvals_(), as(ld_rhs), (is_contiguous_ == true) ? ROOTED : CONTIGUOUS_AND_ROOTED); - } + } + if (debug_level_ > 0) { + if (debug_level_ == 1) { + X->description(); + } else { + Teuchos::RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + X->getMap()->describe(*fancy, Teuchos::VERB_EXTREME); + std::cout << std::endl; + X->describe(*fancy, Teuchos::VERB_EXTREME); + } + } return( 0 ); } @@ -344,6 +365,12 @@ namespace Amesos2 { if( parameterList->isParameter("IsContiguous") ){ is_contiguous_ = parameterList->get("IsContiguous"); } + if( parameterList->isParameter("MessageLevel") ){ + msglvl_ = parameterList->get("MessageLevel"); + } + if( parameterList->isParameter("DebugLevel") ){ + debug_level_ = parameterList->get("DebugLevel"); + } } @@ -467,6 +494,8 @@ PardisoMKL::getValidParameters_impl() const anyNumberParameterEntryValidator(preferred_int, accept_int)); pl->set("IsContiguous", true, "Whether GIDs contiguous"); + pl->set("MessageLevel", 0, "PardisoMKL message level (0 to turn off message, and 1 to turn on message"); + pl->set("DebugLevel", 0, "Debug message level (0 for no message, and >0 for more message"); valid_params = pl; } @@ -483,6 +512,11 @@ PardisoMKL::loadA_impl(EPhase current_phase) #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor convTimer(this->timers_.mtxConvTime_); #endif + if (debug_level_ > 0 && current_phase == NUMFACT) { + if (this->root_) std::cout << "\n == Amesos2_PardisoMKL::loadA_impl(NumFact) ==" << std::endl; + Teuchos::RCP fancy = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); + this->matrixA_->describe(*fancy, Teuchos::VERB_EXTREME); + } // PardisoMKL does not need matrix data in the pre-ordering phase if( current_phase == PREORDERING ) return( false ); @@ -492,7 +526,6 @@ PardisoMKL::loadA_impl(EPhase current_phase) Kokkos::resize(colind_view_, this->globalNumNonZeros_); Kokkos::resize(rowptr_view_, this->globalNumRows_ + 1); } - { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor mtxRedistTimer( this->timers_.mtxRedistTime_ ); @@ -600,10 +633,6 @@ PardisoMKL::set_pardiso_mkl_matrix_type(int_t mtype) template const char* PardisoMKL::name = "PARDISOMKL"; -template -const typename PardisoMKL::int_t -PardisoMKL::msglvl_ = 0; - template const typename PardisoMKL::int_t PardisoMKL::maxfct_ = 1; @@ -612,7 +641,6 @@ template const typename PardisoMKL::int_t PardisoMKL::mnum_ = 1; - } // end namespace Amesos #endif // AMESOS2_PARDISOMKL_DEF_HPP From e0d712f24dc34d92586c95e2b28843c46db0024b Mon Sep 17 00:00:00 2001 From: iyamazaki Date: Wed, 7 Jan 2026 15:15:13 -0700 Subject: [PATCH 05/11] Amesos2: cleanup before re-symbolic for Pardiso & CSS (#14837) * Amesos2: Css free memory before symbolic, if needed Signed-off-by: iyamazaki * Amesos2: test re-symbolic with same solver object Signed-off-by: iyamazaki * Amesos2: cleanup PardisoMKL before re-symbolic Signed-off-by: iyamazaki * Amesos2: guard time prints with verbose for pardiso & css with Amesos2_timer Signed-off-by: iyamazaki --------- Signed-off-by: iyamazaki --- packages/amesos2/src/Amesos2_CssMKL_def.hpp | 36 ++++++++++++- .../amesos2/src/Amesos2_PardisoMKL_decl.hpp | 1 + .../amesos2/src/Amesos2_PardisoMKL_def.hpp | 50 +++++++++++++++++-- packages/amesos2/test/solvers/CMakeLists.txt | 4 +- packages/amesos2/test/solvers/Solver_Test.cpp | 20 ++++++++ .../amesos2/test/solvers/css_mkl_test.xml | 5 ++ .../amesos2/test/solvers/pardiso_mkl_test.xml | 5 ++ 7 files changed, 113 insertions(+), 8 deletions(-) diff --git a/packages/amesos2/src/Amesos2_CssMKL_def.hpp b/packages/amesos2/src/Amesos2_CssMKL_def.hpp index 724b551d95a5..8fb608515304 100644 --- a/packages/amesos2/src/Amesos2_CssMKL_def.hpp +++ b/packages/amesos2/src/Amesos2_CssMKL_def.hpp @@ -140,6 +140,23 @@ namespace Amesos2 { Teuchos::TimeMonitor symbFactTimer( this->timers_.symFactTime_ ); #endif + if (css_initialized_) + { + int_t phase = -1; // release all internal solver memory + void *bdummy, *xdummy; + const MPI_Fint CssComm = CssComm_; + function_map::cluster_sparse_solver( pt_, const_cast(&maxfct_), + const_cast(&mnum_), &mtype_, &phase, &n_, + nzvals_view_.data(), rowptr_view_.data(), + colind_view_.data(), perm_.getRawPtr(), &nrhs_, iparm_, + const_cast(&msglvl_), &bdummy, &xdummy, &CssComm, &error ); + css_initialized_ = false; + if (msglvl_ > 0 && error != 0 && this->matrixA_->getComm()->getRank() == 0) { + std::cout << " CssMKL::symbolicFactorization: clean-up failed with " << error << std::endl; + } + } + + error = 0; int_t phase = 11; // Analysis void *bdummy, *xdummy; const MPI_Fint CssComm = CssComm_; @@ -152,7 +169,11 @@ namespace Amesos2 { check_css_mkl_error(Amesos2::SYMBFACT, error); if (msglvl_ > 0 && this->matrixA_->getComm()->getRank() == 0) { std::cout << " CssMKL::symbolicFactorization done:" << std::endl; +#ifdef HAVE_AMESOS2_TIMERS std::cout << " * Time : " << this->timers_.symFactTime_.totalElapsedTime() << std::endl; +#else + std::cout << " * Time : not enabled" << std::endl; +#endif } // Pardiso only lets you retrieve the total number of factor @@ -190,7 +211,11 @@ namespace Amesos2 { check_css_mkl_error(Amesos2::NUMFACT, error); if (msglvl_ > 0 && this->matrixA_->getComm()->getRank() == 0) { std::cout << " CssMKL::numericFactorization done:" << std::endl; - std::cout << " Time : " << this->timers_.numFactTime_.totalElapsedTime() << std::endl; +#ifdef HAVE_AMESOS2_TIMERS + std::cout << " * Time : " << this->timers_.numFactTime_.totalElapsedTime() << std::endl; +#else + std::cout << " * Time : not enabled" << std::endl; +#endif } return( 0 ); @@ -262,6 +287,15 @@ namespace Amesos2 { as(ld_rhs), Teuchos::ptrInArg(*css_rowmap_)); } + if (msglvl_ > 0 && this->matrixA_->getComm()->getRank() == 0) { + std::cout << " CssMKL::solve done:" << std::endl; +#ifdef HAVE_AMESOS2_TIMERS + std::cout << " * Time : " << this->timers_.vecRedistTime_.totalElapsedTime() + << " + " << this->timers_.solveTime_.totalElapsedTime() << std::endl; +#else + std::cout << " * Time : not enabled" << std::endl; +#endif + } return( 0 ); } diff --git a/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp b/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp index 067a9b4ec7ab..3a6332425be2 100644 --- a/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp +++ b/packages/amesos2/src/Amesos2_PardisoMKL_decl.hpp @@ -275,6 +275,7 @@ namespace Amesos2 { = Meta::or_, std::is_same_v>::value; + bool pardiso_initialized_; bool is_contiguous_; /// The messaging level. Set to 1 if you wish for Pardiso MKL to print statistical info diff --git a/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp b/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp index e2a79bad2edb..80dba3806bae 100644 --- a/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp +++ b/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp @@ -44,6 +44,7 @@ namespace Amesos2 { , n_(Teuchos::as(this->globalNumRows_)) , perm_(this->globalNumRows_) , nrhs_(0) + , pardiso_initialized_(false) , is_contiguous_(true) , msglvl_(0) , debug_level_(0) @@ -83,13 +84,14 @@ namespace Amesos2 { int_t error = 0; void *bdummy, *xdummy; - if( this->root_ ){ + if( this->root_ && pardiso_initialized_){ int_t phase = -1; // release all internal solver memory function_map::pardiso( pt_, const_cast(&maxfct_), const_cast(&mnum_), &mtype_, &phase, &n_, nzvals_view_.data(), rowptr_view_.data(), colind_view_.data(), perm_.getRawPtr(), &nrhs_, iparm_, const_cast(&msglvl_), &bdummy, &xdummy, &error ); + pardiso_initialized_ = false; } check_pardiso_mkl_error(Amesos2::CLEAN, error); @@ -117,19 +119,40 @@ namespace Amesos2 { #ifdef HAVE_AMESOS2_TIMERS Teuchos::TimeMonitor symbFactTimer( this->timers_.symFactTime_ ); #endif - - int_t phase = 11; void *bdummy, *xdummy; + if( pardiso_initialized_){ + int_t phase = -1; // release all internal solver memory + function_map::pardiso( pt_, const_cast(&maxfct_), + const_cast(&mnum_), &mtype_, &phase, &n_, + nzvals_view_.data(), rowptr_view_.data(), + colind_view_.data(), perm_.getRawPtr(), &nrhs_, iparm_, + const_cast(&msglvl_), &bdummy, &xdummy, &error ); + if (msglvl_ > 0 && error != 0) { + std::cout << " PardisoMKL::symbolicFactorization: clean-up failed with " << error << std::endl; + } + pardiso_initialized_ = false; + } + + int_t phase = 11; function_map::pardiso( pt_, const_cast(&maxfct_), const_cast(&mnum_), &mtype_, &phase, &n_, nzvals_view_.data(), rowptr_view_.data(), colind_view_.data(), perm_.getRawPtr(), &nrhs_, iparm_, const_cast(&msglvl_), &bdummy, &xdummy, &error ); + pardiso_initialized_ = true; } - check_pardiso_mkl_error(Amesos2::SYMBFACT, error); + if (msglvl_ > 0 && this->root_) { + std::cout << " PardisoMKL::symbolicFactorization done:" << std::endl; +#ifdef HAVE_AMESOS2_TIMERS + std::cout << " * Time : " << this->timers_.symFactTime_.totalElapsedTime() << std::endl; +#else + std::cout << " * Time : not enabled" << std::endl; +#endif + } + // Pardiso only lets you retrieve the total number of factor // non-zeros, not for each individually. We should document how // such a situation is reported. @@ -158,9 +181,17 @@ namespace Amesos2 { colind_view_.data(), perm_.getRawPtr(), &nrhs_, iparm_, const_cast(&msglvl_), &bdummy, &xdummy, &error ); } - check_pardiso_mkl_error(Amesos2::NUMFACT, error); + if (msglvl_ > 0 && this->root_) { + std::cout << " PardisoMKL::numericFactorization done:" << std::endl; +#ifdef HAVE_AMESOS2_TIMERS + std::cout << " * Time : " << this->timers_.numFactTime_.totalElapsedTime() << std::endl; +#else + std::cout << " * Time : not enabled" << std::endl; +#endif + } + return( 0 ); } @@ -254,6 +285,15 @@ namespace Amesos2 { X->describe(*fancy, Teuchos::VERB_EXTREME); } } + if (msglvl_ > 0 && this->root_) { + std::cout << " PardisoMKL::solve done:" << std::endl; +#ifdef HAVE_AMESOS2_TIMERS + std::cout << " * Time : " << this->timers_.vecRedistTime_.totalElapsedTime() + << " + " << this->timers_.solveTime_.totalElapsedTime() << std::endl; +#else + std::cout << " * Time : not enabled" << std::endl; +#endif + } return( 0 ); } diff --git a/packages/amesos2/test/solvers/CMakeLists.txt b/packages/amesos2/test/solvers/CMakeLists.txt index 72fadf1b0e9f..f41bb2dbf00e 100644 --- a/packages/amesos2/test/solvers/CMakeLists.txt +++ b/packages/amesos2/test/solvers/CMakeLists.txt @@ -376,7 +376,7 @@ IF (${PACKAGE_NAME}_ENABLE_PARDISO_MKL) TRIBITS_ADD_TEST( Solver_Test NAME Pardiso_MKL_Solver_Test - ARGS "--xml-params=pardiso_mkl_test.xml --filedir=${CMAKE_CURRENT_BINARY_DIR}/../matrices/ --multiple-solves --refactor" + ARGS "--xml-params=pardiso_mkl_test.xml --filedir=${CMAKE_CURRENT_BINARY_DIR}/../matrices/ --multiple-solves --refactor --resymbol" STANDARD_PASS_OUTPUT COMM serial mpi ) @@ -394,7 +394,7 @@ IF (${PACKAGE_NAME}_ENABLE_CSS_MKL) TRIBITS_ADD_TEST( Solver_Test NAME Css_MKL_Solver_Test - ARGS "--xml-params=css_mkl_test.xml --filedir=${CMAKE_CURRENT_BINARY_DIR}/../matrices/ --multiple-solves --refactor" + ARGS "--xml-params=css_mkl_test.xml --filedir=${CMAKE_CURRENT_BINARY_DIR}/../matrices/ --multiple-solves --refactor --resymbol" NUM_MPI_PROCS 2 STANDARD_PASS_OUTPUT COMM mpi diff --git a/packages/amesos2/test/solvers/Solver_Test.cpp b/packages/amesos2/test/solvers/Solver_Test.cpp index cb857d2005d6..0f72fe5c75d6 100644 --- a/packages/amesos2/test/solvers/Solver_Test.cpp +++ b/packages/amesos2/test/solvers/Solver_Test.cpp @@ -103,6 +103,12 @@ bool multiple_solves = true; */ bool refactor = false; +/** + * If \c true, then the solution routine will at some point reperform + * symbolic factorization using the same solver object + */ +bool resymbol = false; + /* * Takes the given parameter list and performs the test solves that it describes. * @@ -203,6 +209,7 @@ int main(int argc, char*argv[]) cmdp.setOption("verbosity", &verbosity, "Set verbosity level of output"); cmdp.setOption("multiple-solves","single-solve", &multiple_solves, "Perform multiple solves with different RHS arguments"); cmdp.setOption("refactor","no-refactor", &refactor, "Recompute L and U using a numerically different matrix at some point"); + cmdp.setOption("resymbol","no-resymbol", &resymbol, "Reperform symbolic using the same solver object"); try{ cmdp.parse(argc,argv); } catch (const Teuchos::CommandLineProcessor::HelpPrinted& hp) { @@ -759,6 +766,19 @@ do_solve_routine(const string& solver_name, if( !success ) return success; // bail out early if necessary } + if( resymbol ) { + if (verbosity > 2) { + *fos << endl << " ++ Re Symbolic ++" << std::endl << std::flush; + } + solver->setA(A2, Amesos2::SYMBFACT); + + solver->preOrdering(); + solver->symbolicFactorization(); + solver->numericFactorization(); + solver->solve(outArg(*Xhat), ptrInArg(*b2)); + + success &= checker(x2, Xhat); + } ++style; } diff --git a/packages/amesos2/test/solvers/css_mkl_test.xml b/packages/amesos2/test/solvers/css_mkl_test.xml index c39c1d0a3b7e..90693de334e4 100644 --- a/packages/amesos2/test/solvers/css_mkl_test.xml +++ b/packages/amesos2/test/solvers/css_mkl_test.xml @@ -20,6 +20,11 @@ + + + + + diff --git a/packages/amesos2/test/solvers/pardiso_mkl_test.xml b/packages/amesos2/test/solvers/pardiso_mkl_test.xml index 6f78a4dfb374..9561dec2de8d 100644 --- a/packages/amesos2/test/solvers/pardiso_mkl_test.xml +++ b/packages/amesos2/test/solvers/pardiso_mkl_test.xml @@ -21,6 +21,11 @@ + + + + + From 33c4fcc2c7fd787f26af083e5614e50b5c6c6ddb Mon Sep 17 00:00:00 2001 From: iyamazaki Date: Thu, 11 Dec 2025 19:46:39 -0700 Subject: [PATCH 06/11] Stratimikos: Amesos2 parameters (#14795) * Stratimikos : trying to set Amesos2 parameters Signed-off-by: iyamazaki * Stratimikos : setting Amesos2 (KLU2) parameter from test Signed-off-by: iyamazaki * Stratimikos: update comments about Amesos2 parameter setting Signed-off-by: iyamazaki --------- Signed-off-by: iyamazaki --- ...ra_Amesos2LinearOpWithSolveFactory_def.hpp | 22 ++++++++++--------- ...st_single_amesos2_tpetra_solver_driver.cpp | 5 +++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/stratimikos/adapters/amesos2/src/Thyra_Amesos2LinearOpWithSolveFactory_def.hpp b/packages/stratimikos/adapters/amesos2/src/Thyra_Amesos2LinearOpWithSolveFactory_def.hpp index fe53f668dc14..9e19a209ae0d 100644 --- a/packages/stratimikos/adapters/amesos2/src/Thyra_Amesos2LinearOpWithSolveFactory_def.hpp +++ b/packages/stratimikos/adapters/amesos2/src/Thyra_Amesos2LinearOpWithSolveFactory_def.hpp @@ -205,6 +205,13 @@ void Amesos2LinearOpWithSolveFactory::initializeOp( } } + // Extract and set Amesos2 Parameters + if( paramList_->isSublist(Amesos2_Settings_name) ){ + auto amesos2Params = Teuchos::rcp(new Teuchos::ParameterList(paramList_->sublist(Amesos2_Settings_name))); + amesos2Params->setName("Amesos2"); + amesos2Solver->setParameters(amesos2Params); + } + // Do the initial factorization { THYRA_FUNC_TIME_MONITOR_DIFF("Stratimikos: Amesos2LOWSF:Symbolic", Symbolic); @@ -215,16 +222,6 @@ void Amesos2LinearOpWithSolveFactory::initializeOp( amesos2Solver->numericFactorization(); } - // filter out the Stratimikos adapter parameters and hand - // parameters down into the Solver - const Teuchos::RCP dup_list - = Teuchos::rcp(new Teuchos::ParameterList(*paramList_)); - dup_list->remove(SolverType_name); - dup_list->remove(RefactorizationPolicy_name); - dup_list->remove(ThrowOnPreconditionerInput_name); - dup_list->remove("VerboseObject"); - amesos2Solver->setParameters(dup_list); - // Initialize the LOWS object and we are done! amesos2Op->initialize(fwdOp,fwdOpSrc,amesos2Solver); } @@ -413,6 +410,11 @@ Amesos2LinearOpWithSolveFactory::generateAndGetValidParameters() Amesos2::toString(Amesos2::REPIVOT_ON_REFACTORIZATION)); validParamList->set(ThrowOnPreconditionerInput_name,bool(true)); Teuchos::setupVerboseObjectSublist(&*validParamList); + + // empty Amesos2_Settings parameter list + // (Stratimikos won't validate, but Amesos2 will when a user actually try to set parameters) + RCP amesos2Params = rcp(new ParameterList("Amesos2")); + validParamList->sublist(Amesos2_Settings_name).setParameters(*amesos2Params); } return validParamList; } diff --git a/packages/stratimikos/adapters/amesos2/test/test_single_amesos2_tpetra_solver_driver.cpp b/packages/stratimikos/adapters/amesos2/test/test_single_amesos2_tpetra_solver_driver.cpp index 6450a2e04fa1..9b64455ac511 100644 --- a/packages/stratimikos/adapters/amesos2/test/test_single_amesos2_tpetra_solver_driver.cpp +++ b/packages/stratimikos/adapters/amesos2/test/test_single_amesos2_tpetra_solver_driver.cpp @@ -71,6 +71,11 @@ int main(int argc, char* argv[]) Teuchos::ParameterList amesos2LOWSFPL("Amesos2"); amesos2LOWSFPL.set("Solver Type",solverType); + if (solverType == "KLU2") { + Teuchos::ParameterList & amesos2Params = amesos2LOWSFPL.sublist("Amesos2 Settings"); + Teuchos::ParameterList & klu2Params = amesos2Params.sublist("KLU2"); + klu2Params.set("IsContiguous", false); + } success = Thyra::test_single_amesos2_tpetra_solver( From fb00141f1ca12a4d3e951a5278c6289ecad0a858 Mon Sep 17 00:00:00 2001 From: Alan Williams Date: Mon, 19 Jan 2026 15:44:50 -0700 Subject: [PATCH 07/11] Patch stk destructors (Kokkos decorations) This applies the patch file from the trilinos/github issue https://github.com/spack/spack-packages/pull/2931 Signed-off-by: Alan Williams --- packages/stk/stk_mesh/stk_mesh/base/ConstFieldDataBytes.hpp | 2 +- packages/stk/stk_mesh/stk_mesh/base/FieldData.hpp | 6 +++--- packages/stk/stk_mesh/stk_mesh/base/FieldDataBytes.hpp | 2 +- packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/stk/stk_mesh/stk_mesh/base/ConstFieldDataBytes.hpp b/packages/stk/stk_mesh/stk_mesh/base/ConstFieldDataBytes.hpp index 0642a3c3c85c..d04e3598cd36 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/ConstFieldDataBytes.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/ConstFieldDataBytes.hpp @@ -123,7 +123,7 @@ class ConstFieldDataBytes : public FieldDataBase ConstFieldDataBytes(); ConstFieldDataBytes(EntityRank entityRank, Ordinal fieldOrdinal, const std::string& fieldName, const DataTraits& dataTraits, Layout dataLayout); - virtual ~ConstFieldDataBytes() override = default; + KOKKOS_FUNCTION virtual ~ConstFieldDataBytes() override {} // The AMD ROCm compiler has an "undefined hidden symbol" link error when this copy constructor is defined // below, along with all of the other functions. Not sure why this one is special. diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldData.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldData.hpp index 5bfd29af2466..1ef2277eeac7 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldData.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldData.hpp @@ -99,7 +99,7 @@ class FieldData FieldData(); FieldData(EntityRank entityRank, Ordinal fieldOrdinal, const std::string& fieldName, const DataTraits& dataTraits); - KOKKOS_DEFAULTED_FUNCTION virtual ~FieldData() override = default; + KOKKOS_FUNCTION virtual ~FieldData() override {} FieldData(const FieldData& fieldData, FieldAccessTag accessTag); KOKKOS_DEFAULTED_FUNCTION FieldData(const FieldData& fieldData) = default; @@ -147,7 +147,7 @@ class FieldData FieldData(); FieldData(EntityRank entityRank, Ordinal fieldOrdinal, const std::string& fieldName, const DataTraits& dataTraits); - KOKKOS_DEFAULTED_FUNCTION virtual ~FieldData() override = default; + KOKKOS_FUNCTION virtual ~FieldData() override {} FieldData(const FieldData& fieldData, FieldAccessTag accessTag); KOKKOS_DEFAULTED_FUNCTION FieldData(const FieldData& fieldData) = default; @@ -195,7 +195,7 @@ class FieldData FieldData(); FieldData(const FieldDataBytes& hostFieldBytes, FieldAccessTag accessTag); - KOKKOS_DEFAULTED_FUNCTION virtual ~FieldData() override = default; + KOKKOS_FUNCTION virtual ~FieldData() override {} KOKKOS_DEFAULTED_FUNCTION FieldData(const FieldData& fieldData) = default; KOKKOS_DEFAULTED_FUNCTION FieldData(FieldData&&) = default; diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldDataBytes.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldDataBytes.hpp index fcd9fe03bcc1..9112cbe06c45 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldDataBytes.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldDataBytes.hpp @@ -87,7 +87,7 @@ class FieldDataBytes : public ConstFieldDataBytesmesh_meta_data()); } - virtual ~HostMeshT() override = default; + KOKKOS_FUNCTION virtual ~HostMeshT() override {} HostMeshT(const HostMeshT &) = default; HostMeshT(HostMeshT &&) = default; From 2b156f94ec38a71cdacf4525f80642319d9ec01d Mon Sep 17 00:00:00 2001 From: Ernesto Prudencio Date: Thu, 12 Feb 2026 11:44:39 -0700 Subject: [PATCH 08/11] Conditioning some ETI directives to the situation of 'user requesting trilinoscoupling and tpetra simultaneously' to being 'true'. This change serves the XYCE team in some of its situations (trilinoscouping=ON but tpetra=OFF) where the absence of this conditioning causes the trilinos configuration to fail. Signed-off-by: Ernesto Prudencio --- packages/trilinoscouplings/CMakeLists.txt | 56 ++++++++++++----------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/packages/trilinoscouplings/CMakeLists.txt b/packages/trilinoscouplings/CMakeLists.txt index 125b2148578c..ae188332b4e0 100644 --- a/packages/trilinoscouplings/CMakeLists.txt +++ b/packages/trilinoscouplings/CMakeLists.txt @@ -10,33 +10,35 @@ TRIBITS_PACKAGE(TrilinosCouplings DISABLE_CIRCULAR_REF_DETECTION_FAILURE) # B) Set up package-specific options # -# -# Do explicit template instantiation (ETI) and testing over the same -# Scalar, LocalOrdinal, and GlobalOrdinal types as for Tpetra objects. -# However, by default, exclude all of Tpetra's GlobalOrdinal (GO) -# types from the ETI list. This is because Tpetra adds all GO ETI -# types to the list of Scalar ETI types. (This lets it implement -# certain communication patterns involving e.g., Vector.) -# - -# Make sure that Tpetra actually defined these variables, even if they -# are empty. -ASSERT_DEFINED(TpetraCore_ETI_SCALARS_NO_ORDS) -ASSERT_DEFINED(TpetraCore_ETI_LORDS) -ASSERT_DEFINED(TpetraCore_ETI_GORDS) -ASSERT_DEFINED(TpetraCore_ETI_NODES) - -# Promote these variables to be visible outside of the "scope" of this -# directory -- e.g., to other packages. -GLOBAL_SET(TrilinosCouplings_ETI_SCALARS ${TpetraCore_ETI_SCALARS_NO_ORDS}) -GLOBAL_SET(TrilinosCouplings_ETI_LORDS ${TpetraCore_ETI_LORDS}) -GLOBAL_SET(TrilinosCouplings_ETI_GORDS ${TpetraCore_ETI_GORDS}) -GLOBAL_SET(TrilinosCouplings_ETI_NODES ${TpetraCore_ETI_NODES}) - -TRIBITS_ADD_EXPLICIT_INSTANTIATION_OPTION() - -# we use this for testing as well as library eti, so do it regardless -TRIBITS_ADD_ETI_SUPPORT() +IF(HAVE_TRILINOSCOUPLINGS_TPETRA) + # + # Do explicit template instantiation (ETI) and testing over the same + # Scalar, LocalOrdinal, and GlobalOrdinal types as for Tpetra objects. + # However, by default, exclude all of Tpetra's GlobalOrdinal (GO) + # types from the ETI list. This is because Tpetra adds all GO ETI + # types to the list of Scalar ETI types. (This lets it implement + # certain communication patterns involving e.g., Vector.) + # + + # Make sure that Tpetra actually defined these variables, even if they + # are empty. + ASSERT_DEFINED(TpetraCore_ETI_SCALARS_NO_ORDS) + ASSERT_DEFINED(TpetraCore_ETI_LORDS) + ASSERT_DEFINED(TpetraCore_ETI_GORDS) + ASSERT_DEFINED(TpetraCore_ETI_NODES) + + # Promote these variables to be visible outside of the "scope" of this + # directory -- e.g., to other packages. + GLOBAL_SET(TrilinosCouplings_ETI_SCALARS ${TpetraCore_ETI_SCALARS_NO_ORDS}) + GLOBAL_SET(TrilinosCouplings_ETI_LORDS ${TpetraCore_ETI_LORDS}) + GLOBAL_SET(TrilinosCouplings_ETI_GORDS ${TpetraCore_ETI_GORDS}) + GLOBAL_SET(TrilinosCouplings_ETI_NODES ${TpetraCore_ETI_NODES}) + + TRIBITS_ADD_EXPLICIT_INSTANTIATION_OPTION() + + # we use this for testing as well as library eti, so do it regardless + TRIBITS_ADD_ETI_SUPPORT() +ENDIF() TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_Newp_swahili From 1e5cc368d6104089442e874b8f0b091075d718fd Mon Sep 17 00:00:00 2001 From: Heidi Thornquist Date: Wed, 3 Dec 2025 12:44:12 -0700 Subject: [PATCH 09/11] The Clang 16 compiler throws a build error for a complex-valued scalar comparison with 0.0: TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == 0.0, std::runtime_error,... The variable 'pivot' is a scalar template argument that can be a complex number, so this comparison should be to the zero provided by Teuchos::ScalarTraits class. Signed-off-by: Heidi Thornquist --- .../Tpetra_CrsSingletonFilter_LinearProblem_def.hpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_CrsSingletonFilter_LinearProblem_def.hpp b/packages/tpetra/core/src/Tpetra_CrsSingletonFilter_LinearProblem_def.hpp index ac7521c11a9b..ac7038ab42d3 100644 --- a/packages/tpetra/core/src/Tpetra_CrsSingletonFilter_LinearProblem_def.hpp +++ b/packages/tpetra/core/src/Tpetra_CrsSingletonFilter_LinearProblem_def.hpp @@ -965,6 +965,8 @@ void CrsSingletonFilter_LinearProblem TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(Problem->getLHS() == Teuchos::null, std::runtime_error, "Need a LHS."); + Scalar zero = Teuchos::ScalarTraits::zero(); + // Generate reduced row and column maps if (SingletonsDetected()) { ReducedMatrixRowMap_ = GenerateReducedMap(FullMatrixRowMap(), RowMapColors_, 0); @@ -1057,7 +1059,7 @@ void CrsSingletonFilter_LinearProblem GetRow(i, NumEntries, Values, localIndices); // Get current row if (NumEntries == 1) { Scalar pivot = Values[0]; - TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == 0.0, std::runtime_error, + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == zero, std::runtime_error, "Encountered zero row, unable to continue."); // Should improve this comparison to zero. LocalOrdinal indX = localIndices[0]; for (LocalOrdinal j = 0; j < NumVectors; j++) { @@ -1070,7 +1072,7 @@ void CrsSingletonFilter_LinearProblem for (size_t j = 0; j < NumEntries; j++) { if (localIndices[j] == targetCol) { Scalar pivot = Values[j]; - TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == 0.0, std::runtime_error, + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == zero, std::runtime_error, "Encountered zero column, unable to continue."); // Should improve this comparison to zero. ColSingletonPivotLIDs_[ColSingletonCounter] = j; ColSingletonPivots_[ColSingletonCounter] = pivot; @@ -1258,6 +1260,8 @@ void CrsSingletonFilter_LinearProblem TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(Problem->getLHS() == Teuchos::null, std::runtime_error, "Need a LHS."); + Scalar zero = Teuchos::ScalarTraits::zero(); + if (SingletonsDetected()) { // Create pointer to Full RHS, LHS Teuchos::RCP FullRHS = FullProblem()->getRHS(); @@ -1303,7 +1307,7 @@ void CrsSingletonFilter_LinearProblem GetRow(i, NumEntries, Values, localIndices); // Get current row if (NumEntries == 1) { Scalar pivot = Values[0]; - TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == 0.0, std::runtime_error, + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == zero, std::runtime_error, "Encountered zero row, unable to continue."); // Should improve this comparison to zero. LocalOrdinal indX = localIndices[0]; for (LocalOrdinal j = 0; j < NumVectors; j++) { @@ -1315,7 +1319,7 @@ void CrsSingletonFilter_LinearProblem LocalOrdinal j = ColSingletonPivotLIDs_[ColSingletonCounter]; Scalar pivot = Values[j]; - TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == 0.0, std::runtime_error, + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(pivot == zero, std::runtime_error, "Encountered zero column, unable to continue."); // Should improve this comparison to zero. ColSingletonPivots_[ColSingletonCounter] = pivot; ColSingletonCounter++; From 8a1419abbd0c4cdc7490dedb100d5166fdc195f2 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Tue, 31 Mar 2026 07:25:01 -0600 Subject: [PATCH 10/11] Belos: Add missing dependency Signed-off-by: Christian Glusa --- packages/belos/cmake/Dependencies.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/belos/cmake/Dependencies.cmake b/packages/belos/cmake/Dependencies.cmake index 2a5c16896cfe..19b71c30af5e 100644 --- a/packages/belos/cmake/Dependencies.cmake +++ b/packages/belos/cmake/Dependencies.cmake @@ -1,4 +1,4 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Teuchos) +SET(LIB_REQUIRED_DEP_PACKAGES Teuchos Kokkos) SET(LIB_OPTIONAL_DEP_PACKAGES Epetra Tpetra Xpetra Thyra AztecOO Triutils KokkosKernels) SET(TEST_REQUIRED_DEP_PACKAGES Tpetra) SET(TEST_OPTIONAL_DEP_PACKAGES Galeri Triutils EpetraExt Ifpack ML AztecOO) From 66578ae8a99981672335a0b50a8525e654c5373b Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Mon, 1 Dec 2025 07:21:49 -0700 Subject: [PATCH 11/11] Panzer: Disable memUtils test Signed-off-by: Christian Glusa --- packages/panzer/core/test/memUtils.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/panzer/core/test/memUtils.cpp b/packages/panzer/core/test/memUtils.cpp index 41c50eafc86a..e9620d90d8bb 100644 --- a/packages/panzer/core/test/memUtils.cpp +++ b/packages/panzer/core/test/memUtils.cpp @@ -57,19 +57,22 @@ namespace panzer delete[] newDouble; } // end of TEUCHOS_UNIT_TEST() + // CAG: Test disabled since it failed on some systems. In all these failures + // the peak memory usage was below the current one. + // Test that the peak memory usage is always greater than the current memory // usage. - TEUCHOS_UNIT_TEST(memUtils, currentVsPeak) - { - Teuchos::RCP > comm = - Teuchos::DefaultComm::getComm(); + // TEUCHOS_UNIT_TEST(memUtils, currentVsPeak) + // { + // Teuchos::RCP > comm = + // Teuchos::DefaultComm::getComm(); - // Get the current and peak memory usage. - MemUsage mem = getMemoryUsage(*comm); + // // Get the current and peak memory usage. + // MemUsage mem = getMemoryUsage(*comm); - // Test that the peak usage is greater than the current usage. - TEST_COMPARE(mem.peakMin, >=, mem.currMin); - TEST_COMPARE(mem.peakMax, >=, mem.currMax); - TEST_COMPARE(mem.peakTot, >=, mem.currTot); - } // end of TEUCHOS_UNIT_TEST() + // // Test that the peak usage is greater than the current usage. + // TEST_COMPARE(mem.peakMin, >=, mem.currMin); + // TEST_COMPARE(mem.peakMax, >=, mem.currMax); + // TEST_COMPARE(mem.peakTot, >=, mem.currTot); + // } // end of TEUCHOS_UNIT_TEST() } // end namespace panzer