xe: sdpa, ggemm, gmlp: avoid copies

Author: atkassen

src/gpu/intel/gated_mlp/micro_horz.cpp

@@ -197,7 +197,7 @@ status_t micro_horz_t::pd_t::init_microkernels(
                     && (problem.Tb != gemmstone::Type::invalid),
             status::unimplemented, "Incompatible A/B types in uGEMM.");
 
-    auto problem_wgu = problem;
+    auto problem_wgu = std::move(problem);
     problem_wgu.A.layout = gemmstone::MatrixLayout::T;
     problem_wgu.B.layout = gemmstone::MatrixLayout::Pr;
     problem_wgu.C.layout = gemmstone::MatrixLayout::T;

src/gpu/intel/gated_mlp/ref.hpp

@@ -161,7 +161,7 @@ struct ref_t : public primitive_t {
 
     status_t execute(const exec_ctx_t &ctx) const override {
         auto prep_weights_and_run
-                = [&](exec_args_t &args, int idx,
+                = [&](exec_args_t &&args, int idx,
                           const std::shared_ptr<impl::primitive_t> &prim) {
             args[DNNL_ARG_WEIGHTS] = ctx.args().at(idx);
             if (!pd()->attr()->scales_.has_default_values(idx))
@@ -201,7 +201,8 @@ struct ref_t : public primitive_t {
             exec_args_t args;
             args[DNNL_ARG_SRC] = ctx.args().at(DNNL_ARG_SRC);
             args[DNNL_ARG_DST] = memory_arg_t {inter_src_mem.get(), false};
-            CHECK(prep_weights_and_run(args, DNNL_ARG_WEIGHTS_UP, gemm_up_));
+            CHECK(prep_weights_and_run(
+                    std::move(args), DNNL_ARG_WEIGHTS_UP, gemm_up_));
         } while (false);
         do {
             exec_args_t args;
@@ -211,14 +212,14 @@ struct ref_t : public primitive_t {
             args[DNNL_ARG_ATTR_MULTIPLE_POST_OP(1) | DNNL_ARG_SRC_1]
                     = memory_arg_t {inter_src_mem.get(), true};
             CHECK(prep_weights_and_run(
-                    args, DNNL_ARG_WEIGHTS_GATE, gemm_gate_));
+                    std::move(args), DNNL_ARG_WEIGHTS_GATE, gemm_gate_));
         } while (false);
         do {
             exec_args_t args;
             args[DNNL_ARG_SRC] = memory_arg_t {inter_wei_mem.get(), true};
             args[DNNL_ARG_DST] = ctx.args().at(DNNL_ARG_DST);
             CHECK(prep_weights_and_run(
-                    args, DNNL_ARG_WEIGHTS_DOWN, gemm_down_));
+                    std::move(args), DNNL_ARG_WEIGHTS_DOWN, gemm_down_));
         } while (false);
 
         return status::success;

src/gpu/intel/gemm/jit/gen_kernel.cpp

@@ -166,7 +166,7 @@ status_t gen_desc_t::finalize(const char *tags) {
         problem_.beta = stringToScalar(val);
 
         ovr_strategy = ss.str().substr(ss.tellg()); // remaining string
-        parseStrategy(ovr_strategy.c_str(), hw_, problem_, strategy_);
+        parseStrategy(ovr_strategy, hw_, problem_, strategy_);
 
         // TODO: override derived values in aux_params_ in a way that's
         // consistent with the kernel evaluator (typically requires extra

src/gpu/intel/gemm/jit/generator/microkernel/shim.cpp

@@ -231,7 +231,7 @@ std::string generateShim(const Package &package, HostLanguage language,
     auto nargs = int(pargs.size());
 
     /* Match up protocol settings with microkernel settings */
-    auto psettings = package.protocol.settings();
+    const auto &psettings = package.protocol.settings();
     auto settings = matchProtocol(psettings, package.settings);
 
     /* Collect actual argument types */

src/gpu/intel/gemm/jit/generator/microkernel_selector.cpp

@@ -147,7 +147,7 @@ GEMMOptions GEMMOptions::transpose() const {
     return ret;
 }
 
-std::string strategyToString(HW hw, GEMMProblem problem, GEMMStrategy strategy) {
+std::string strategyToString(HW hw, const GEMMProblem &problem, const GEMMStrategy &strategy) {
     std::stringstream ss;
     ss << problem.toString() << " "
        << std::to_string(strategy.unroll[LoopM])

src/gpu/intel/gemm/jit/generator/strategy_parser.cpp

@@ -170,7 +170,7 @@ static void getTiling(std::stringstream &s, MatrixAddressingStrategy &astrategy)
     }
 }
 
-void parseStrategy(const char *str, HW hw, const GEMMProblem &problem, GEMMStrategy &strategy)
+void parseStrategy(const std::string &str, HW hw, const GEMMProblem &problem, GEMMStrategy &strategy)
 {
     std::stringstream s(str);
     s.imbue(std::locale::classic());

src/gpu/intel/gemm/jit/include/gemmstone/strategy_parser.hpp

@@ -25,7 +25,7 @@
 
 GEMMSTONE_NAMESPACE_START
 
-void parseStrategy(const char *str, ngen::HW hw, const GEMMProblem &problem, GEMMStrategy &strategy);
+void parseStrategy(const std::string &str, ngen::HW hw, const GEMMProblem &problem, GEMMStrategy &strategy);
 
 void adjustStrategy(ngen::HW hw, const GEMMProblem &problem, GEMMStrategy &strategy, const char *tags = nullptr);
 

src/gpu/intel/matmul/grouped_micro_gemm.cpp

@@ -190,7 +190,7 @@ status_t grouped_micro_gemm_t::pd_t::init_microkernels(impl::engine_t *engine) {
             Scalar alpha(a), beta(b);
             std::string strategyString;
             std::getline(ss >> std::ws, strategyString);
-            parseStrategy(strategyString.c_str(), hw, problem, strat);
+            parseStrategy(strategyString, hw, problem, strat);
             adjustStrategy(hw, problem, strat);
         }
         strategyGRFs_ = strat.GRFs;

src/gpu/intel/sdpa/micro.cpp

@@ -625,7 +625,7 @@ status_t micro_bwd_t::pd_t::init_conf_microkernels(impl::engine_t *engine) {
     ukernel_params.sizes_kq = {heuristic_sizes};
 
     /* Set up GEMMProblem structure for second GEMM: V * S  */
-    auto problem_vs = std::move(problem);
+    auto problem_vs = problem;
     problem_vs.Tc = problem_vs.Ts
             = (vs_acc_dt() == data_type::f16) ? Type::f16 : Type::f32;
 
@@ -723,7 +723,7 @@ status_t micro_bwd_t::pd_t::init_conf_microkernels(impl::engine_t *engine) {
     ukernel_params.opts_qdSt = {opts_qdSt};
 
     // dS * K
-    auto problem_ktq = problem;
+    auto problem_ktq = std::move(problem);
     problem_ktq.Ta_ext
             = convert_dnnl_to_kernel_type(desc()->key_md()->data_type);
 
@@ -1180,7 +1180,7 @@ status_t micro_fwd_params_t::get_kernel_ctx(
             ss >> strat.unroll[1];
             std::string strategyString;
             std::getline(ss >> std::ws, strategyString);
-            parseStrategy(strategyString.c_str(), hw, problem_kq, strat);
+            parseStrategy(strategyString, hw, problem_kq, strat);
             adjustStrategy(hw, problem_kq, strat);
         }
     };
@@ -1208,7 +1208,7 @@ status_t micro_fwd_params_t::get_kernel_ctx(
             ss >> strat.unroll[1];
             std::string strategyString;
             std::getline(ss >> std::ws, strategyString);
-            parseStrategy(strategyString.c_str(), hw, problem_vs, strat);
+            parseStrategy(strategyString, hw, problem_vs, strat);
             adjustStrategy(hw, problem_vs, strat);
         }
     };

tests/gtests/internals/test_sdpa.cpp

@@ -1677,7 +1677,7 @@ std::chrono::nanoseconds prim_sdpa_quant_bwd(const sdpa_dims_t &p,
     strm.wait();
     if (dropout_mask_fwd_out != nullptr && p.dropout.enabled()
             && p.dropout.has_output_mask()) {
-        *dropout_mask_fwd_out = softmax_dropout_mask;
+        *dropout_mask_fwd_out = std::move(softmax_dropout_mask);
     }
 
 #if DEBUG_PRINT_MEM