diff --git a/CMakeLists.txt b/CMakeLists.txt index f038a12bb0f..3813b53d91c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,42 @@ macro(opm-simulators_prereqs_hook) # NVCC as for the rest. In the case that NVCC does not support # that compiler it will error out. if(NOT CMAKE_DISABLE_FIND_PACKAGE_CUDA AND NOT CONVERT_CUDA_TO_HIP) + # do I need this? + if(CMAKE_BUILD_TYPE) + set(_flags_suffix "_${CMAKE_BUILD_TYPE}") + endif() + if(NOT DEFINED ENV{CUDAHOSTCXX} AND NOT DEFINED CMAKE_CUDA_HOST_COMPILER AND + (NOT CMAKE_CUDA_FLAGS${_flags_suffix} OR NOT CMAKE_CUDA_FLAGS${_flags_suffix} MATCHES ".*-ccbin .*")) + message(STATUS "Setting CUDA host compiler CMAKE_CUDA_HOST_COMPILER to ${CMAKE_CXX_COMPILER} to " + "prevent incompatibilities. Note that this might report that there " + "is not CUDA compiler if your system's CUDA compiler does not support " + "${CMAKE_CXX_COMPILER}.") + # check_language does not seem to care about ${CMAKE_CUDA_FLAGS} or $(CUDA_NVCC_FLAGS}. + # Hence we set CMAKE_CUDA_HOST_COMPILER to our C++ compiler. + # In check_language(CUDA) we will get an error if we in addition put + # "-ccbin ${CMAKE_CXX_COMPILER}" into CMAKE_CUDA_FLAGS. It results + # in "${NVCC} -ccbin=${CMAKE_CXX_COMPILER} -ccbin ${CMAKE_CXX_COMPILER}" + # which causes nvcc to abort + set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) + set(ENV{CUDAHOSTCXX} ${CMAKE_CUDA_HOST_COMPILER}) # The only thing honored by check_language(CUDA)! + endif() + include(CheckLanguage) + check_language(CUDA) + if(CMAKE_CUDA_COMPILER) + # OPTIONAL is ignored. Hence the magic above to check whether enabling CUDA works + enable_language(CUDA) + set(CUDA_FOUND ON) + else() + message(FATAL_ERROR "CUDA support requested but no suitable CUDA compiler found. " + "Use -DCMAKE_DISABLE_FIND_PACKAGE_CUDA=ON to disable CUDA support.") + endif() + if(CUDA_FOUND AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS "9.0") + set(CUDA_FOUND OFF) + message(WARNING + "Deactivating CUDA as we require version 9.0 or newer." + " Found only CUDA version ${CUDA_VERSION}." + ) + endif() if(CUDA_FOUND) find_package(CUDAToolkit REQUIRED) target_link_libraries(opmsimulators @@ -364,8 +400,10 @@ macro(opm-simulators_sources_hook) # cuda warns when constxpr functions are used in kernels. # since the entire stl is constexpr ..., we enable relaxed flag set_source_files_properties( + opm/simulators/linalg/gpuistl/GpuBlackoilIntensiveQuantitiesDispatcher.cu tests/gpuistl/test_blackoilfluidstategpu.cu tests/gpuistl/test_gpu_ad.cu + tests/gpuistl/test_gpu_ecl_thermal_law_manager.cu tests/gpuistl/test_gpu_linear_two_phase_material.cu tests/gpuistl/test_gpuPvt.cu tests/gpuistl/test_gpuBlackOilFluidSystem.cu @@ -490,6 +528,7 @@ macro(opm-simulators_tests_hook) cuVector_operations deviceBlockOperations gpu_ad + gpu_ecl_thermal_law_manager gpu_linear_two_phase_material gpu_resources gpu_safe_call @@ -515,6 +554,7 @@ macro(opm-simulators_tests_hook) primary_variables_gpu solver_adapter throw_macros_on_gpu + blackoilintensivequantities_gpu ) if(TEST ${test}) set_tests_properties(${test} @@ -681,6 +721,9 @@ macro(opm-simulators_targets_hook) target_sources(test_RestartSerialization PRIVATE $) target_sources(test_glift1 PRIVATE $) target_sources(test_tpsa_localresidual PRIVATE $) + if (TARGET test_blackoilintensivequantities_gpu) + target_sources(test_blackoilintensivequantities_gpu PRIVATE $) + endif() if(MPI_FOUND) target_sources(test_chopstep PRIVATE $) endif() @@ -711,6 +754,7 @@ macro(opm-simulators_targets_hook) gaswater_saltprec_energy gaswater_saltprec_vapwat gaswater_solvent + gpu micp oilwater oilwater_brine @@ -725,14 +769,25 @@ macro(opm-simulators_targets_hook) ) foreach(OBJ ${COMMON_MODELS} ${FLOW_MODELS} ${FLOW_VARIANT_MODELS}) + set(SOURCE_FILE "flow/flow_${OBJ}.cpp") + + if(OBJ STREQUAL "gpu") + if (CONVERT_CUDA_TO_HIP) + set(SOURCE_FILE "flow/flow_${OBJ}.hip") + else() + set(SOURCE_FILE "flow/flow_${OBJ}.cu") + endif() + endif() + opm_add_library( TARGET flow_lib${OBJ} TYPE OBJECT SOURCES - flow/flow_${OBJ}.cpp + ${SOURCE_FILE} ) + simulators_add_target_options(TARGET flow_lib${OBJ}) list(APPEND FLOW_TGTS $) list(APPEND FLOWMODELS_PREFIXED flow_${OBJ}) diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index 6075ba58577..fbc8a8608e1 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -337,9 +337,12 @@ if(CUDA_FOUND OR hip_FOUND) ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/FlexibleSolverWrapper.cpp) ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg FlexibleSolver_gpu_instantiate.cpp) ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg PreconditionerFactory_gpu_instantiate.cpp) + ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg GpuBlackoilIntensiveQuantitiesDispatcher.cu) # HEADERS + ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg GpuBlackoilIntensiveQuantitiesDispatcher.hpp) + ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg GpuFlowGasWaterEnergyTypeTags.hpp) ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/autotuner.hpp) ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/coloringAndReorderingUtils.hpp) ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/gpu_safe_call.hpp) @@ -570,6 +573,7 @@ if(CUDA_FOUND OR hip_FOUND) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_safe_conversion.cpp) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_solver_adapter.cpp) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_ad.cu) + ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_ecl_thermal_law_manager.cu) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_linear_two_phase_material.cu) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpuPvt.cu) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_smart_pointers.cu) @@ -587,6 +591,8 @@ if(CUDA_FOUND OR hip_FOUND) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_primary_variables_gpu.cu) endif() ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_MiniMatrix.cu) + ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_blackoilintensivequantities_gpu.cu) + # Boost < 1.75 + nvcc = trouble in this test if(Boost_VERSION VERSION_GREATER 1.74) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_MiniVector.cu) @@ -595,6 +601,34 @@ if(CUDA_FOUND OR hip_FOUND) if(MPI_FOUND) ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_GpuOwnerOverlapCopy.cpp) endif() + + # for loop providing the flag --expt-relaxed-constexpr to fix some cuda issues with constexpr + if(NOT CONVERT_CUDA_TO_HIP) + set(CU_FILES_NEEDING_RELAXED_CONSTEXPR + tests/gpuistl/test_gpu_ad.cu + tests/gpuistl/test_gpu_linear_two_phase_material.cu + tests/gpuistl/test_gpuPvt.cu + tests/gpuistl/test_gpuBlackOilFluidSystem.cu + tests/gpuistl/test_GpuSparseMatrix.cu + tests/gpuistl/test_GpuSparseTable.cu + tests/gpuistl/test_blackoilfluidstategpu.cu + flow/flow_gpu.cu + ) + + foreach(file ${CU_FILES_NEEDING_RELAXED_CONSTEXPR}) + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "--expt-relaxed-constexpr") + endforeach() + + set(CU_FILES_NEEDING_FPERMISSIVE + tests/gpuistl/test_primary_variables_gpu.cu + ) + + foreach(file ${CU_FILES_NEEDING_FPERMISSIVE}) + # Certain structures in OPM requires the -fpermissive flag to compile with nvcc, + # this enables this for the specific files + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "-fpermissive --expt-relaxed-constexpr -Xcompiler=-fpermissive") + endforeach() + endif() endif() if(USE_GPU_BRIDGE) @@ -707,6 +741,7 @@ list (APPEND TEST_DATA_FILES tests/data/test_stokes2c.dgf tests/data/test_stokes2cni.dgf tests/data/waterair.dgf + tests/very_simple_deck.DATA ) @@ -966,6 +1001,7 @@ list (APPEND PUBLIC_HEADER_FILES opm/simulators/flow/FacePropertiesTPSA_impl.hpp opm/simulators/flow/FemCpGridCompat.hpp opm/simulators/flow/FIBlackoilModel.hpp + opm/simulators/flow/SimplifiedGpuBlackOilModel.hpp opm/simulators/flow/FIPContainer.hpp opm/simulators/flow/FlowBaseProblemProperties.hpp opm/simulators/flow/FlowBaseVanguard.hpp @@ -1014,6 +1050,7 @@ list (APPEND PUBLIC_HEADER_FILES opm/simulators/flow/RFTContainer.hpp opm/simulators/flow/RSTConv.hpp opm/simulators/flow/RegionPhasePVAverage.hpp + opm/simulators/flow/SimplifiedFlowProblemGPU.hpp opm/simulators/flow/SimulatorConvergenceOutput.hpp opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp opm/simulators/flow/SimulatorFullyImplicitBlackoil_impl.hpp diff --git a/flow/flow_gaswater_energy.cpp b/flow/flow_gaswater_energy.cpp index f41a022e6ba..12d11c1ff2b 100644 --- a/flow/flow_gaswater_energy.cpp +++ b/flow/flow_gaswater_energy.cpp @@ -22,74 +22,12 @@ #include #include -#include #include #include #include -#include -#include -#include - -namespace Opm { -namespace Properties { -namespace TTag { -struct FlowGasWaterEnergyProblem { - using InheritsFrom = std::tuple; -}; -} - -template -struct Linearizer { using type = TpfaLinearizer; }; - -template -struct LocalResidual { using type = BlackOilLocalResidualTPFA; }; - -template -struct EnableDiffusion { static constexpr bool value = true; }; - -template -struct EnableDispersion { static constexpr bool value = true; }; - -template -struct EnergyModuleType -{ static constexpr EnergyModules value = EnergyModules::FullyImplicitThermal; }; - -template -struct EnableDisgasInWater { - static constexpr bool value = true; -}; - -template -struct EnableVapwat { - static constexpr bool value = true; -}; - -//! The indices required by the model -template -struct Indices -{ -private: - // it is unfortunately not possible to simply use 'TypeTag' here because this leads - // to cyclic definitions of some properties. if this happens the compiler error - // messages unfortunately are *really* confusing and not really helpful. - using BaseTypeTag = TTag::FlowProblem; - using FluidSystem = GetPropType; - static constexpr EnergyModules energyModuleType = getPropValue(); - static constexpr int numEnergyVars = energyModuleType == EnergyModules::FullyImplicitThermal; -public: - using type = BlackOilTwoPhaseIndices(), - getPropValue(), - getPropValue(), - numEnergyVars, - getPropValue(), - getPropValue(), - /*PVOffset=*/0, - /*disabledCompIdx=*/FluidSystem::oilCompIdx, - getPropValue()>; -}; -}} +#include namespace Opm { diff --git a/flow/flow_gpu.cu b/flow/flow_gpu.cu new file mode 100644 index 00000000000..73bdad29732 --- /dev/null +++ b/flow/flow_gpu.cu @@ -0,0 +1,67 @@ +/* + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ +#include "config.h" + +// For now flow_gpu is developed to support SPE11 simulations, that is 2-phase gas-water flow with +// thermal effects The goal is to support both property-evaluation and matrix assembly on the GPU, +// in addition to the linear solver which already works on the GPU. The CPU would still have to +// manage well contributions, although there are no wells in SPE11, only source-terms. + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace Opm +{ + +int +flowGasWaterEnergyMainGPU(int argc, char** argv, bool outputCout, bool outputFiles) +{ + // we always want to use the default locale, and thus spare us the trouble + // with incorrect locale settings. + resetLocale(); + + FlowMain mainfunc { + argc, argv, outputCout, outputFiles}; + return mainfunc.execute(); +} + +int +flowGasWaterEnergyMainGPUStandalone(int argc, char** argv) +{ + using TypeTag = Properties::TTag::FlowGasWaterEnergyProblemGPU; + auto mainObject = std::make_unique<::Opm::Main>(argc, argv); + auto ret = mainObject->runStatic(); + // Destruct mainObject as the destructor calls MPI_Finalize! + mainObject.reset(); + return ret; +} + +} // namespace Opm diff --git a/flow/flow_gpu.hip b/flow/flow_gpu.hip new file mode 100644 index 00000000000..2d931fc3077 --- /dev/null +++ b/flow/flow_gpu.hip @@ -0,0 +1,67 @@ +/* + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ +#include "config.h" + +// For now flow_gpu is developed to support SPE11 simulations, that is 2-phase gas-water flow with +// thermal effects The goal is to support both property-evaluation and matrix assembly on the GPU, +// in addition to the linear solver which already works on the GPU. The CPU would still have to +// manage well contributions, although there are no wells in SPE11, only source-terms. + +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace Opm +{ + +int +flowGasWaterEnergyMainGPU(int argc, char** argv, bool outputCout, bool outputFiles) +{ + // we always want to use the default locale, and thus spare us the trouble + // with incorrect locale settings. + resetLocale(); + + FlowMain mainfunc { + argc, argv, outputCout, outputFiles}; + return mainfunc.execute(); +} + +int +flowGasWaterEnergyMainGPUStandalone(int argc, char** argv) +{ + using TypeTag = Properties::TTag::FlowGasWaterEnergyProblemGPU; + auto mainObject = std::make_unique<::Opm::Main>(argc, argv); + auto ret = mainObject->runStatic(); + // Destruct mainObject as the destructor calls MPI_Finalize! + mainObject.reset(); + return ret; +} + +} // namespace Opm diff --git a/flow/flow_gpu.hpp b/flow/flow_gpu.hpp new file mode 100644 index 00000000000..9d2706e0882 --- /dev/null +++ b/flow/flow_gpu.hpp @@ -0,0 +1,74 @@ +/* + This file is part of the Open Porous Media project (OPM). + + OPM is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OPM is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OPM. If not, see . +*/ +#ifndef FLOW_GPU_HIP_HPP +#define FLOW_GPU_HIP_HPP + +#include + +namespace Opm +{ + +namespace Properties +{ + namespace TTag + { + // FlowGasWaterEnergyProblemGPU is declared in FlowGasWaterEnergyTypeTag.hpp + // (InheritsFrom = FlowGasWaterEnergyProblem). The template below maps it + // to FlowGasWaterEnergyProblemGPUTrue for GPU-storage variants. + + template