GPU Simulation POC

CMakeLists.txt

@@ -198,6 +198,42 @@ macro(opm-simulators_prereqs_hook)
   # NVCC as for the rest. In the case that NVCC does not support
   # that compiler it will error out.
   if(NOT CMAKE_DISABLE_FIND_PACKAGE_CUDA AND NOT CONVERT_CUDA_TO_HIP)
+    # do I need this?
+    if(CMAKE_BUILD_TYPE)
+      set(_flags_suffix "_${CMAKE_BUILD_TYPE}")
+    endif()
+    if(NOT DEFINED ENV{CUDAHOSTCXX} AND NOT DEFINED CMAKE_CUDA_HOST_COMPILER AND
+        (NOT CMAKE_CUDA_FLAGS${_flags_suffix} OR NOT CMAKE_CUDA_FLAGS${_flags_suffix} MATCHES ".*-ccbin .*"))
+      message(STATUS "Setting CUDA host compiler CMAKE_CUDA_HOST_COMPILER to ${CMAKE_CXX_COMPILER} to "
+        "prevent incompatibilities. Note that this might report that there "
+        "is not CUDA compiler if your system's CUDA compiler does not support "
+        "${CMAKE_CXX_COMPILER}.")
+      # check_language does not seem to care about ${CMAKE_CUDA_FLAGS} or $(CUDA_NVCC_FLAGS}.
+      # Hence we set CMAKE_CUDA_HOST_COMPILER to our C++ compiler.
+      # In check_language(CUDA) we will get an error if we in addition put
+      # "-ccbin ${CMAKE_CXX_COMPILER}" into CMAKE_CUDA_FLAGS. It results
+      # in "${NVCC} -ccbin=${CMAKE_CXX_COMPILER}   -ccbin ${CMAKE_CXX_COMPILER}"
+      # which causes nvcc to abort
+      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+      set(ENV{CUDAHOSTCXX} ${CMAKE_CUDA_HOST_COMPILER}) # The only thing honored by check_language(CUDA)!
+    endif()
+    include(CheckLanguage)
+    check_language(CUDA)
+    if(CMAKE_CUDA_COMPILER)
+      # OPTIONAL is ignored. Hence the magic above to check whether enabling CUDA works
+      enable_language(CUDA)
+      set(CUDA_FOUND ON)
+    else()
+      message(FATAL_ERROR "CUDA support requested but no suitable CUDA compiler found. "
+        "Use -DCMAKE_DISABLE_FIND_PACKAGE_CUDA=ON to disable CUDA support.")
+    endif()
+    if(CUDA_FOUND AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS "9.0")
+      set(CUDA_FOUND OFF)
+      message(WARNING
+        "Deactivating CUDA as we require version 9.0 or newer."
+        " Found only CUDA version ${CUDA_VERSION}."
+      )
+    endif()
     if(CUDA_FOUND)
       find_package(CUDAToolkit REQUIRED)
       target_link_libraries(opmsimulators
@@ -364,8 +400,10 @@ macro(opm-simulators_sources_hook)
     # cuda warns when constxpr functions are used in kernels.
     # since the entire stl is constexpr ..., we enable relaxed flag
     set_source_files_properties(
+      opm/simulators/linalg/gpuistl/GpuBlackoilIntensiveQuantitiesDispatcher.cu
       tests/gpuistl/test_blackoilfluidstategpu.cu
       tests/gpuistl/test_gpu_ad.cu
+      tests/gpuistl/test_gpu_ecl_thermal_law_manager.cu
       tests/gpuistl/test_gpu_linear_two_phase_material.cu
       tests/gpuistl/test_gpuPvt.cu
       tests/gpuistl/test_gpuBlackOilFluidSystem.cu
@@ -490,6 +528,7 @@ macro(opm-simulators_tests_hook)
                  cuVector_operations
                  deviceBlockOperations
                  gpu_ad
+                 gpu_ecl_thermal_law_manager
                  gpu_linear_two_phase_material
                  gpu_resources
                  gpu_safe_call
@@ -515,6 +554,7 @@ macro(opm-simulators_tests_hook)
                  primary_variables_gpu
                  solver_adapter
                  throw_macros_on_gpu
+                 blackoilintensivequantities_gpu
            )
       if(TEST ${test})
         set_tests_properties(${test}
@@ -681,6 +721,9 @@ macro(opm-simulators_targets_hook)
   target_sources(test_RestartSerialization PRIVATE $<TARGET_OBJECTS:moduleVersion>)
   target_sources(test_glift1 PRIVATE $<TARGET_OBJECTS:moduleVersion>)
   target_sources(test_tpsa_localresidual PRIVATE $<TARGET_OBJECTS:moduleVersion>)
+  if (TARGET test_blackoilintensivequantities_gpu)
+    target_sources(test_blackoilintensivequantities_gpu PRIVATE $<TARGET_OBJECTS:moduleVersion>)
+  endif()
   if(MPI_FOUND)
     target_sources(test_chopstep PRIVATE $<TARGET_OBJECTS:moduleVersion>)
   endif()
@@ -711,6 +754,7 @@ macro(opm-simulators_targets_hook)
     gaswater_saltprec_energy
     gaswater_saltprec_vapwat
     gaswater_solvent
+    gpu
     micp
     oilwater
     oilwater_brine
@@ -725,14 +769,25 @@ macro(opm-simulators_targets_hook)
   )
 
   foreach(OBJ ${COMMON_MODELS} ${FLOW_MODELS} ${FLOW_VARIANT_MODELS})
+    set(SOURCE_FILE "flow/flow_${OBJ}.cpp")
+
+    if(OBJ STREQUAL "gpu")
+      if (CONVERT_CUDA_TO_HIP)
+        set(SOURCE_FILE "flow/flow_${OBJ}.hip")
+      else()
+        set(SOURCE_FILE "flow/flow_${OBJ}.cu")
+      endif()
+    endif()
+
     opm_add_library(
       TARGET
         flow_lib${OBJ}
       TYPE
         OBJECT
       SOURCES
-        flow/flow_${OBJ}.cpp
+        ${SOURCE_FILE}
     )
+
     simulators_add_target_options(TARGET flow_lib${OBJ})
     list(APPEND FLOW_TGTS $<TARGET_OBJECTS:flow_lib${OBJ}>)
     list(APPEND FLOWMODELS_PREFIXED flow_${OBJ})

CMakeLists_files.cmake

@@ -337,9 +337,12 @@ if(CUDA_FOUND OR hip_FOUND)
   ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/FlexibleSolverWrapper.cpp)
   ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg FlexibleSolver_gpu_instantiate.cpp)
   ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg PreconditionerFactory_gpu_instantiate.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg GpuBlackoilIntensiveQuantitiesDispatcher.cu)
 
 
   # HEADERS
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg GpuBlackoilIntensiveQuantitiesDispatcher.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg GpuFlowGasWaterEnergyTypeTags.hpp)
   ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/autotuner.hpp)
   ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/coloringAndReorderingUtils.hpp)
   ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/gpu_safe_call.hpp)
@@ -570,6 +573,7 @@ if(CUDA_FOUND OR hip_FOUND)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_safe_conversion.cpp)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_solver_adapter.cpp)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_ad.cu)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_ecl_thermal_law_manager.cu)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_linear_two_phase_material.cu)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpuPvt.cu)
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_gpu_smart_pointers.cu)
@@ -587,6 +591,8 @@ if(CUDA_FOUND OR hip_FOUND)
     ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_primary_variables_gpu.cu)
   endif()
   ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_MiniMatrix.cu)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_blackoilintensivequantities_gpu.cu)
+
   # Boost < 1.75 + nvcc = trouble in this test
   if(Boost_VERSION VERSION_GREATER 1.74)
     ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_MiniVector.cu)
@@ -595,6 +601,34 @@ if(CUDA_FOUND OR hip_FOUND)
   if(MPI_FOUND)
     ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_GpuOwnerOverlapCopy.cpp)
   endif()
+
+  # for loop providing the flag --expt-relaxed-constexpr to fix some cuda issues with constexpr
+  if(NOT CONVERT_CUDA_TO_HIP)
+    set(CU_FILES_NEEDING_RELAXED_CONSTEXPR
+      tests/gpuistl/test_gpu_ad.cu
+      tests/gpuistl/test_gpu_linear_two_phase_material.cu
+      tests/gpuistl/test_gpuPvt.cu
+      tests/gpuistl/test_gpuBlackOilFluidSystem.cu
+      tests/gpuistl/test_GpuSparseMatrix.cu
+      tests/gpuistl/test_GpuSparseTable.cu
+      tests/gpuistl/test_blackoilfluidstategpu.cu
+      flow/flow_gpu.cu
+    )
+
+    foreach(file ${CU_FILES_NEEDING_RELAXED_CONSTEXPR})
+        set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "--expt-relaxed-constexpr")
+    endforeach()
+
+    set(CU_FILES_NEEDING_FPERMISSIVE
+      tests/gpuistl/test_primary_variables_gpu.cu
+    )
+
+    foreach(file ${CU_FILES_NEEDING_FPERMISSIVE})
+      # Certain structures in OPM requires the -fpermissive flag to compile with nvcc,
+      # this enables this for the specific files
+      set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "-fpermissive --expt-relaxed-constexpr -Xcompiler=-fpermissive")
+    endforeach()
+  endif()
 endif()
 
 if(USE_GPU_BRIDGE)
@@ -707,6 +741,7 @@ list (APPEND TEST_DATA_FILES
   tests/data/test_stokes2c.dgf
   tests/data/test_stokes2cni.dgf
   tests/data/waterair.dgf
+  tests/very_simple_deck.DATA
   )
 
 
@@ -966,6 +1001,7 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/flow/FacePropertiesTPSA_impl.hpp
   opm/simulators/flow/FemCpGridCompat.hpp
   opm/simulators/flow/FIBlackoilModel.hpp
+  opm/simulators/flow/SimplifiedGpuBlackOilModel.hpp
   opm/simulators/flow/FIPContainer.hpp
   opm/simulators/flow/FlowBaseProblemProperties.hpp
   opm/simulators/flow/FlowBaseVanguard.hpp
@@ -1014,6 +1050,7 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/flow/RFTContainer.hpp
   opm/simulators/flow/RSTConv.hpp
   opm/simulators/flow/RegionPhasePVAverage.hpp
+  opm/simulators/flow/SimplifiedFlowProblemGPU.hpp
   opm/simulators/flow/SimulatorConvergenceOutput.hpp
   opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp
   opm/simulators/flow/SimulatorFullyImplicitBlackoil_impl.hpp

flow/flow_gaswater_energy.cpp

@@ -22,74 +22,12 @@
 #include <flow/flow_gaswater_energy.hpp>
 
 #include <opm/material/common/ResetLocale.hpp>
-#include <opm/models/blackoil/blackoiltwophaseindices.hh>
 
 #include <opm/grid/CpGrid.hpp>
 #include <opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp>
 #include <opm/simulators/flow/Main.hpp>
 
-#include <opm/models/blackoil/blackoillocalresidualtpfa.hh>
-#include <opm/models/discretization/common/tpfalinearizer.hh>
-#include <opm/material/thermal/EnergyModuleType.hpp>
-
-namespace Opm {
-namespace Properties {
-namespace TTag {
-struct FlowGasWaterEnergyProblem {
-    using InheritsFrom = std::tuple<FlowProblem>;
-};
-}
-
-template<class TypeTag>
-struct Linearizer<TypeTag, TTag::FlowGasWaterEnergyProblem> { using type = TpfaLinearizer<TypeTag>; };
-
-template<class TypeTag>
-struct LocalResidual<TypeTag, TTag::FlowGasWaterEnergyProblem> { using type = BlackOilLocalResidualTPFA<TypeTag>; };
-
-template<class TypeTag>
-struct EnableDiffusion<TypeTag, TTag::FlowGasWaterEnergyProblem> { static constexpr bool value = true; };
-
-template<class TypeTag>
-struct EnableDispersion<TypeTag, TTag::FlowGasWaterEnergyProblem> { static constexpr bool value = true; };
-
-template<class TypeTag>
-struct EnergyModuleType<TypeTag, TTag::FlowGasWaterEnergyProblem>
-{ static constexpr EnergyModules value = EnergyModules::FullyImplicitThermal; };
-
-template<class TypeTag>
-struct EnableDisgasInWater<TypeTag, TTag::FlowGasWaterEnergyProblem> {
-    static constexpr bool value = true;
-};
-
-template<class TypeTag>
-struct EnableVapwat<TypeTag, TTag::FlowGasWaterEnergyProblem> {
-    static constexpr bool value = true;
-};
-
-//! The indices required by the model
-template<class TypeTag>
-struct Indices<TypeTag, TTag::FlowGasWaterEnergyProblem>
-{
-private:
-    // it is unfortunately not possible to simply use 'TypeTag' here because this leads
-    // to cyclic definitions of some properties. if this happens the compiler error
-    // messages unfortunately are *really* confusing and not really helpful.
-    using BaseTypeTag = TTag::FlowProblem;
-    using FluidSystem = GetPropType<BaseTypeTag, Properties::FluidSystem>;
-    static constexpr EnergyModules energyModuleType = getPropValue<TypeTag, Properties::EnergyModuleType>();
-    static constexpr int numEnergyVars = energyModuleType == EnergyModules::FullyImplicitThermal;
-public:
-    using type = BlackOilTwoPhaseIndices<getPropValue<TypeTag, Properties::EnableSolvent>(),
-                                         getPropValue<TypeTag, Properties::EnableExtbo>(),
-                                         getPropValue<TypeTag, Properties::EnablePolymer>(),
-                                         numEnergyVars,
-                                         getPropValue<TypeTag, Properties::EnableFoam>(),
-                                         getPropValue<TypeTag, Properties::EnableBrine>(),
-                                         /*PVOffset=*/0,
-                                         /*disabledCompIdx=*/FluidSystem::oilCompIdx,
-                                         getPropValue<TypeTag, Properties::EnableBioeffects>()>;
-};
-}}
+#include <opm/simulators/flow/FlowGasWaterEnergyTypeTag.hpp>
 
 namespace Opm {
 

flow/flow_gpu.cu

@@ -0,0 +1,67 @@
+/*
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "config.h"
+
+// For now flow_gpu is developed to support SPE11 simulations, that is 2-phase gas-water flow with
+// thermal effects The goal is to support both property-evaluation and matrix assembly on the GPU,
+// in addition to the linear solver which already works on the GPU. The CPU would still have to
+// manage well contributions, although there are no wells in SPE11, only source-terms.
+
+#include <flow/flow_gpu.hpp>
+
+#include <opm/common/utility/gpuDecorators.hpp>
+
+#include <opm/material/common/ResetLocale.hpp>
+#include <opm/material/thermal/EnergyModuleType.hpp>
+#include <opm/models/blackoil/blackoiltwophaseindices.hh>
+
+#include <opm/grid/CpGrid.hpp>
+#include <opm/simulators/flow/FlowGasWaterEnergyTypeTag.hpp>
+#include <opm/simulators/flow/Main.hpp>
+#include <opm/simulators/flow/SimplifiedGpuBlackOilModel.hpp>
+#include <opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp>
+
+#include <opm/models/blackoil/blackoillocalresidualtpfa.hh>
+#include <opm/models/discretization/common/tpfalinearizer.hh>
+
+namespace Opm
+{
+
+int
+flowGasWaterEnergyMainGPU(int argc, char** argv, bool outputCout, bool outputFiles)
+{
+    // we always want to use the default locale, and thus spare us the trouble
+    // with incorrect locale settings.
+    resetLocale();
+
+    FlowMain<Properties::TTag::FlowGasWaterEnergyProblemGPU> mainfunc {
+        argc, argv, outputCout, outputFiles};
+    return mainfunc.execute();
+}
+
+int
+flowGasWaterEnergyMainGPUStandalone(int argc, char** argv)
+{
+    using TypeTag = Properties::TTag::FlowGasWaterEnergyProblemGPU;
+    auto mainObject = std::make_unique<::Opm::Main>(argc, argv);
+    auto ret = mainObject->runStatic<TypeTag>();
+    // Destruct mainObject as the destructor calls MPI_Finalize!
+    mainObject.reset();
+    return ret;
+}
+
+} // namespace Opm