From 9bdf8c10af8e039d02f06021f5fa7e1dde629b4b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 5 Mar 2026 12:25:50 -0800 Subject: [PATCH 1/4] Fix SIGSEGV from Comm. tools on HIP builds --- .../mesh/mpiCommunications/CommunicationTools.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp index 6fd4ed29383..678c0c49fc9 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp +++ b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp @@ -1019,11 +1019,15 @@ void CommunicationTools::asyncSendRecv( stdVector< NeighborCommunicator > & neig parallelDeviceEvents & events ) { GEOS_MARK_FUNCTION; +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } - +#else + GEOS_UNUSED_VAR( onDevice ); + GEOS_UNUSED_VAR( events ); +#endif // could swap this to test and make this function call async as well, only launch the sends/recvs for // those we've already recv'd sizing for, go back to some usefule compute / launch some other compute, then @@ -1162,10 +1166,14 @@ void CommunicationTools::finalizeUnpack( ObjectManagerBase & manager, // poll mpi for completion then wait 10 nanoseconds 6,000,000,000 times (60 sec timeout) GEOS_ASYNC_WAIT( 6000000000, 10, asyncUnpack( manager, neighbors, icomm, onDevice, events ) ); +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } +#else + GEOS_UNUSED_VAR( onDevice ); +#endif MpiWrapper::waitAll( icomm.size(), icomm.mpiSendBufferSizeRequest(), @@ -1187,10 +1195,14 @@ void CommunicationTools::finalizeUnpack( MeshLevel & mesh, // poll mpi for completion then wait 10 nanoseconds 6,000,000,000 times (60 sec timeout) GEOS_ASYNC_WAIT( 6000000000, 10, asyncUnpack( mesh, neighbors, icomm, onDevice, events, op ) ); +#if !defined( GEOS_USE_HIP ) if( onDevice ) { waitAllDeviceEvents( events ); } +#else + GEOS_UNUSED_VAR( onDevice ); +#endif MpiWrapper::waitAll( icomm.size(), icomm.mpiSendBufferSizeRequest(), From 99c290a48f1b08694184330f9f6e61290ba40a68 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 5 Mar 2026 12:26:45 -0800 Subject: [PATCH 2/4] Switch default to vendor SpGEMM in hypre for HIP builds --- .../linearAlgebra/interfaces/hypre/HypreInterface.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index 6c94111d6a1..f5f291c772f 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -55,7 +55,11 @@ void HypreInterface::initialize() HYPRE_Initialize(); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP HYPRE_SetExecutionPolicy( HYPRE_EXEC_DEVICE ); +#if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA HYPRE_SetSpGemmUseVendor( 0 ); +#else + HYPRE_SetSpGemmUseVendor( 1 ); +#endif HYPRE_DeviceInitialize(); #endif HYPRE_SetMemoryLocation( hypre::memoryLocation ); From 3e9397720dee5bce3cd56bd9ab982ab2c24594e4 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 5 Mar 2026 12:27:37 -0800 Subject: [PATCH 3/4] Version header minor fixes --- src/coreComponents/mainInterface/version.cpp | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/coreComponents/mainInterface/version.cpp b/src/coreComponents/mainInterface/version.cpp index ba26582fab5..668d0103b46 100644 --- a/src/coreComponents/mainInterface/version.cpp +++ b/src/coreComponents/mainInterface/version.cpp @@ -72,13 +72,13 @@ void outputVersionInfo() GEOS_LOG_RANK_0( "GEOS version: " << getVersion() ); - GEOS_LOG_RANK_0( " - c++ compiler: " << getCppCompilerIdString() ); + GEOS_LOG_RANK_0( " - C++ compiler: " << getCppCompilerIdString() ); std::string const gpuCompilerIdString = getGpuCompilerIdString(); GEOS_LOG_RANK_0_IF( !gpuCompilerIdString.empty(), gpuCompilerIdString ); #if defined(_OPENMP) - GEOS_LOG_RANK_0( " - openmp version: " << _OPENMP ); + GEOS_LOG_RANK_0( " - OpenMP version: " << _OPENMP ); #endif #if defined(GEOS_USE_MPI) @@ -107,11 +107,11 @@ void outputVersionInfo() #endif #if defined(umpire_VERSION) - GEOS_LOG_RANK_0( " - umpire version: " << STRINGIZE( umpire_VERSION ) ); + GEOS_LOG_RANK_0( " - Umpire version: " << STRINGIZE( umpire_VERSION ) ); #endif #if defined(chai_VERSION) - GEOS_LOG_RANK_0( " - chai version: " << STRINGIZE( chai_VERSION ) ); + GEOS_LOG_RANK_0( " - CHAI version: " << STRINGIZE( chai_VERSION ) ); #endif #if defined(adiak_VERSION) @@ -119,7 +119,7 @@ void outputVersionInfo() #endif #if defined(caliper_VERSION) - GEOS_LOG_RANK_0( " - caliper version: " << STRINGIZE( caliper_VERSION ) ); + GEOS_LOG_RANK_0( " - Caliper version: " << STRINGIZE( caliper_VERSION ) ); #endif #if defined(metis_VERSION) @@ -127,19 +127,19 @@ void outputVersionInfo() #endif #if defined(parmetis_VERSION) - GEOS_LOG_RANK_0( " - PARAMETIS version: " << STRINGIZE( parmetis_VERSION ) ); + GEOS_LOG_RANK_0( " - PARMETIS version: " << STRINGIZE( parmetis_VERSION ) ); #endif #if defined(scotch_VERSION) - GEOS_LOG_RANK_0( " - scotch version: " << STRINGIZE( scotch_VERSION ) ); + GEOS_LOG_RANK_0( " - Scotch version: " << STRINGIZE( scotch_VERSION ) ); #endif #if defined(superlu_dist_VERSION) - GEOS_LOG_RANK_0( " - superlu_dist version: " << STRINGIZE( superlu_dist_VERSION ) ); + GEOS_LOG_RANK_0( " - SuperLU_Dist version: " << STRINGIZE( superlu_dist_VERSION ) ); #endif #if defined(suitesparse_VERSION) - GEOS_LOG_RANK_0( " - suitesparse version: " << STRINGIZE( suitesparse_VERSION ) ); + GEOS_LOG_RANK_0( " - SuiteSparse version: " << STRINGIZE( suitesparse_VERSION ) ); #endif #if defined(hypre_VERSION) @@ -151,7 +151,7 @@ void outputVersionInfo() #endif #if defined(petsc_VERSION) - GEOS_LOG_RANK_0( " - petsc version: " << STRINGIZE( petsc_VERSION ) ); + GEOS_LOG_RANK_0( " - PETSc version: " << STRINGIZE( petsc_VERSION ) ); #endif #if defined(Python3_VERSION) @@ -167,11 +167,11 @@ void outputVersionInfo() defined(GEOS_USE_HYPRE) && \ ( GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CPU ) GEOS_LOG_RANK_0( "" ); - GEOS_LOG_RANK_0( "**************************************************" ); - GEOS_LOG_RANK_0( "* WARNING!!! *" ); - GEOS_LOG_RANK_0( "* *" ); - GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but not HYPRE! *" ); - GEOS_LOG_RANK_0( "**************************************************" ); + GEOS_LOG_RANK_0( "*******************************************************" ); + GEOS_LOG_RANK_0( "* WARNING!! WARNING!! WARNING!! *" ); + GEOS_LOG_RANK_0( "* *" ); + GEOS_LOG_RANK_0( "* GEOS has GPU support enabled, but HYPRE does not! *" ); + GEOS_LOG_RANK_0( "*******************************************************" ); GEOS_LOG_RANK_0( "" ); #endif From 86974da2af6f9ea0bf25242d4c9c230d6af92ef2 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 5 Mar 2026 21:33:21 -0800 Subject: [PATCH 4/4] Add hipMemInfo APIs --- src/coreComponents/common/MemoryInfos.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/coreComponents/common/MemoryInfos.cpp b/src/coreComponents/common/MemoryInfos.cpp index ac44f996d9d..13031f71c5e 100644 --- a/src/coreComponents/common/MemoryInfos.cpp +++ b/src/coreComponents/common/MemoryInfos.cpp @@ -50,6 +50,11 @@ MemoryInfos::MemoryInfos( umpire::MemoryResourceTraits::resource_type resourceTy case umpire::MemoryResourceTraits::resource_type::um: #if defined( GEOS_USE_CUDA ) cudaMemGetInfo( &m_availableMemory, &m_totalMemory ); + #elif defined( GEOS_USE_HIP ) + { + hipError_t const err = hipMemGetInfo( &m_availableMemory, &m_totalMemory ); + GEOS_WARNING_IF( err != hipSuccess, GEOS_FMT( "HIP error found: {} at {}:{}", hipGetErrorString( err ), __FILE__, __LINE__ ) ); + } #else GEOS_WARNING( "Unknown device physical memory size getter for this compiler." ); m_physicalMemoryHandled = 0;