CUTLASS 3.6.0 (#1850)

* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>

tools/profiler/src/conv2d_operation_profiler.cu

@@ -51,10 +51,10 @@ namespace profiler {
 /////////////////////////////////////////////////////////////////////////////////////////////////
 
 /// Ctor
-Conv2dOperationProfiler::Conv2dOperationProfiler(Options const &options): 
+Conv2dOperationProfiler::Conv2dOperationProfiler(Options const &options):
   OperationProfiler(
     options,
-    library::OperationKind::kConv2d, 
+    library::OperationKind::kConv2d,
     {
       {ArgumentTypeID::kEnumerated, {"conv_kind"}, "Convolutional operator (fprop, dgrad, wgrad)"},
       {ArgumentTypeID::kInteger, {"n", "input_n"}, "Input N dimension of the Conv2d problem space"},
@@ -165,13 +165,13 @@ int64_t Conv2dOperationProfiler::Conv2dProblem::flops(
 
   int64_t flops_mainloop_ = int64_t(mnk.m()) * mnk.n() * mnk.k() * 2;
   int64_t flops_epilogue_ = int64_t(mnk.m()) * int64_t(mnk.n()) * 2;
-  
+
   // Adjust mainloop flop for dgrad strided
   if (operation_desc.conv_kind == library::ConvKind::kDgrad) {
     flops_mainloop_ = flops_mainloop_ / (stride_h * stride_w);
   }
   int64_t flops_total_ = flops_mainloop_ + flops_epilogue_;
-  
+
   //complex-valued support
   switch (operation_desc.tile_description.math_instruction.math_operation) {
   case library::MathOperationID::kMultiplyAddComplex:
@@ -188,14 +188,14 @@ int64_t Conv2dOperationProfiler::Conv2dProblem::flops(
 
 /// Extracts the problem dimensions
 Status Conv2dOperationProfiler::initialize_configuration(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
   ProblemSpace const &problem_space,
   ProblemSpace::Problem const &problem) {
 
-  library::ConvDescription const &operation_desc = 
+  library::ConvDescription const &operation_desc =
     static_cast<library::ConvDescription const &>(operation->description());
 
   if (!arg_as_int(problem_.n, "n", problem_space, problem)) {
@@ -207,7 +207,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
     // default value
     problem_.h = 16;
   }
-  
+
   if (!arg_as_int(problem_.w, "w", problem_space, problem)) {
     // default value
     problem_.w = 16;
@@ -227,7 +227,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
     // default value
     problem_.r = 3;
   }
-  
+
   if (!arg_as_int(problem_.s, "s", problem_space, problem)) {
     // default value
     problem_.s = 3;
@@ -280,14 +280,14 @@ Status Conv2dOperationProfiler::initialize_configuration(
   // cutlass profiler sets p and q which are cuDNN compliant.                           //
   //                                                                                    //
   ////////////////////////////////////////////////////////////////////////////////////////
-  // set convolution output p 
+  // set convolution output p
   if (!arg_as_int(problem_.p, "p", problem_space, problem)) {
     // default value (set using cudnn formula for output height, when p is not provided)
     problem_.p = (
-                    problem_.h + 
-                    2 * problem_.pad_h - 
+                    problem_.h +
+                    2 * problem_.pad_h -
                     ((problem_.r - 1) * problem_.dilation_h + 1)
-                 ) / (problem_.stride_h) 
+                 ) / (problem_.stride_h)
                 + 1;
   }
 
@@ -295,10 +295,10 @@ Status Conv2dOperationProfiler::initialize_configuration(
   if (!arg_as_int(problem_.q, "q", problem_space, problem)) {
     // default value (set using cudnn formula for output width, when q is not provided)
     problem_.q = (
-                    problem_.w + 
-                    2 * problem_.pad_w - 
+                    problem_.w +
+                    2 * problem_.pad_w -
                     ((problem_.s - 1) * problem_.dilation_w + 1)
-                 ) / (problem_.stride_w) 
+                 ) / (problem_.stride_w)
                 + 1;
   }
   /////////////////////////////////////////////////////////////////////////////////////////
@@ -313,7 +313,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
     // default value
     problem_.split_k_slices = 1;
   }
-  
+
   if (!arg_as_ConvModeID(problem_.conv_mode, "conv_mode", problem_space, problem)) {
     // default value
     problem_.conv_mode = library::ConvModeID::kCrossCorrelation;
@@ -345,24 +345,24 @@ Status Conv2dOperationProfiler::initialize_configuration(
   }
 
   if (!arg_as_scalar(
-    problem_.alpha, 
-    operation_desc.element_epilogue, 
-    "alpha", 
-    problem_space, 
+    problem_.alpha,
+    operation_desc.element_epilogue,
+    "alpha",
+    problem_space,
     problem)) {
 
     if (!cast_from_double(problem_.alpha, operation_desc.element_epilogue, 1)) {
       return Status::kErrorInternal;
     }
   }
-  
+
   if (!arg_as_scalar(
-    problem_.beta, 
-    operation_desc.element_epilogue, 
-    "beta", 
-    problem_space, 
+    problem_.beta,
+    operation_desc.element_epilogue,
+    "beta",
+    problem_space,
     problem)) {
-    
+
     if (!cast_from_double(problem_.beta, operation_desc.element_epilogue, 0)) {
       return Status::kErrorInternal;
     }
@@ -389,7 +389,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
                                                 int(problem_.split_k_slices),
                                                 int(problem_.groups)
                                               );
-  
+
   conv_workspace_.configuration.split_k_mode = static_cast<conv::SplitKMode>(static_cast<int>(problem_.split_k_mode));
 
   conv_workspace_.set_stride_vector(
@@ -420,7 +420,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
 /// Initializes the performance result
 void Conv2dOperationProfiler::initialize_result_(
   PerformanceResult &result,
-  Options const &options,  
+  Options const &options,
   library::ConvDescription const &operation_desc,
   ProblemSpace const &problem_space) {
 
@@ -432,15 +432,15 @@ void Conv2dOperationProfiler::initialize_result_(
   result.arguments.resize(problem_space.rank());
 
   set_argument(result, "Activation", problem_space,
-    std::string(library::to_string(operation_desc.activation().element)) 
+    std::string(library::to_string(operation_desc.activation().element))
     + ":" + library::to_string(operation_desc.activation().layout));
 
   set_argument(result, "Filter", problem_space,
-    std::string(library::to_string(operation_desc.filter().element)) 
+    std::string(library::to_string(operation_desc.filter().element))
     + ":" + library::to_string(operation_desc.filter().layout));
 
   set_argument(result, "Output", problem_space,
-    std::string(library::to_string(operation_desc.output().element)) 
+    std::string(library::to_string(operation_desc.output().element))
     + ":" + library::to_string(operation_desc.output().layout));
 
   set_argument(result, "conv_kind", problem_space, library::to_string(operation_desc.conv_kind));
@@ -455,7 +455,7 @@ void Conv2dOperationProfiler::initialize_result_(
   set_argument(result, "k", problem_space, problem_.k);
   set_argument(result, "r", problem_space, problem_.r);
   set_argument(result, "s", problem_space, problem_.s);
-  
+
   set_argument(result, "p", problem_space, problem_.p);
   set_argument(result, "q", problem_space, problem_.q);
 
@@ -470,11 +470,11 @@ void Conv2dOperationProfiler::initialize_result_(
   set_argument(result, "dilation_h", problem_space, problem_.dilation_h);
   set_argument(result, "dilation_w", problem_space, problem_.dilation_w);
 
-  set_argument(result, "split_k_mode", problem_space, 
+  set_argument(result, "split_k_mode", problem_space,
     std::string(library::to_string(problem_.split_k_mode)));
   set_argument(result, "split_k_slices", problem_space, problem_.split_k_slices);
 
-  set_argument(result, "conv_mode", problem_space, 
+  set_argument(result, "conv_mode", problem_space,
     std::string(library::to_string(problem_.conv_mode)));
 
   set_argument(result, "alpha", problem_space,
@@ -483,19 +483,19 @@ void Conv2dOperationProfiler::initialize_result_(
   set_argument(result, "beta", problem_space,
     library::lexical_cast(problem_.beta, operation_desc.element_epilogue));
 
-  set_argument(result, "eq_gemm_provider", problem_space, 
+  set_argument(result, "eq_gemm_provider", problem_space,
     std::string(library::to_string(problem_.eq_gemm_provider)));
 
   OperationProfiler::initialize_result_(result, operation_desc, problem_space);
 
   // Bytes of activation, filter, and output tensors
-  int64_t activation_bytes = int64_t(library::sizeof_bits(operation_desc.activation().element) / 8) * 
+  int64_t activation_bytes = int64_t(library::sizeof_bits(operation_desc.activation().element) / 8) *
     conv_workspace_.configuration.problem_size.activation_size();
 
-  int64_t filter_bytes = int64_t(library::sizeof_bits(operation_desc.filter().element) / 8) * 
+  int64_t filter_bytes = int64_t(library::sizeof_bits(operation_desc.filter().element) / 8) *
     conv_workspace_.configuration.problem_size.filter_size();
 
-  int64_t output_bytes = int64_t(library::sizeof_bits(operation_desc.output().element) / 8) * 
+  int64_t output_bytes = int64_t(library::sizeof_bits(operation_desc.output().element) / 8) *
     conv_workspace_.configuration.problem_size.output_size();
 
   // Bytes of activation, filter, and output tensors
@@ -511,14 +511,14 @@ void Conv2dOperationProfiler::initialize_result_(
 
 /// Initialize reduction problem dimensions and library::Operation
 bool Conv2dOperationProfiler::initialize_reduction_configuration_(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
   ProblemSpace const &problem_space,
   ProblemSpace::Problem const &problem) {
 
-  library::ConvDescription const &conv_desc = 
+  library::ConvDescription const &conv_desc =
     static_cast<library::ConvDescription const &>(operation->description());
 
   library::ConvKind const &conv_kind = conv_desc.conv_kind;
@@ -545,14 +545,14 @@ bool Conv2dOperationProfiler::initialize_reduction_configuration_(
   conv_workspace_.reduction_configuration.ldd =
       conv_workspace_.configuration.stride_c[tensor_c_stride_idx];
 
-  // find reduction operation 
+  // find reduction operation
   library::ReductionFunctionalKey reduction_key(
     library::Provider::kCUTLASS,
-    conv_desc.tile_description.math_instruction.element_accumulator,  // element workspace 
+    conv_desc.tile_description.math_instruction.element_accumulator,  // element workspace
     conv_desc.tile_description.math_instruction.element_accumulator,  // element accumulator
     conv_desc.C.element,                                              // element output
     conv_desc.element_epilogue                                        // element compute
-  ); 
+  );
 
 #if 0// debug print to check which reduction instance is selected
     std::cout << reduction_key << "\n";
@@ -562,7 +562,7 @@ bool Conv2dOperationProfiler::initialize_reduction_configuration_(
   if(reduction_it == Singleton::get().operation_table.reduction_operations.end()) {
 
     return false;
-  }    
+  }
 
   // initialize reduction operation required for parallel split-k conv2d operator
   reduction_op_ = reduction_it->second;
@@ -574,13 +574,24 @@ bool Conv2dOperationProfiler::initialize_reduction_configuration_(
 
 /// Initializes workspace
 Status Conv2dOperationProfiler::initialize_workspace(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
   ProblemSpace const &problem_space,
   ProblemSpace::Problem const &problem) {
 
+  if (options.device.devices.size() != 1) {
+    throw std::runtime_error("This operation profiler only supports a single "
+                             "device.");
+  }
+
+  cudaError_t result;
+  result = cudaSetDevice(options.device.device_id(0));
+  if (result != cudaSuccess) {
+    throw std::runtime_error("cudaSetDevice() failed.");
+  }
+
   // initialize conv2d underlying operation to handle parallel reduction
   library::Operation const* underlying_operation = operation;
 
@@ -590,15 +601,15 @@ Status Conv2dOperationProfiler::initialize_workspace(
     }
   }
 
-  library::ConvDescription const &operation_desc = 
+  library::ConvDescription const &operation_desc =
     static_cast<library::ConvDescription const &>(underlying_operation->description());
 
   // Compute the number of copies of the problem to avoid L2 camping.
   if (!options.profiling.workspace_count) {
     int64_t bytes = problem_.bytes(operation_desc);
-    if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
+    if (bytes < 3 * int64_t(options.device.properties[0].l2CacheSize)) {
       conv_workspace_.problem_count =
-        1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
+        1 + int((3 * int64_t(options.device.properties[0].l2CacheSize)) / bytes);
     }
     else {
       conv_workspace_.problem_count = 1;
@@ -611,7 +622,7 @@ Status Conv2dOperationProfiler::initialize_workspace(
 
   if (options.execution_mode != ExecutionMode::kDryRun) {
     int seed_shift = 0;
-    conv_workspace_.A = device_context.allocate_tensor(
+    conv_workspace_.A = device_context.allocate_and_initialize_tensor(
       options,
       "A",
       operation_desc.A.element,
@@ -619,10 +630,11 @@ Status Conv2dOperationProfiler::initialize_workspace(
       problem_.extent_a(operation_desc.conv_kind),
       conv_workspace_.configuration.stride_a,
       conv_workspace_.problem_count,
-      seed_shift++
+      seed_shift++,
+      0 // device_index
     );
 
-    conv_workspace_.B = device_context.allocate_tensor(
+    conv_workspace_.B = device_context.allocate_and_initialize_tensor(
       options,
       "B",
       operation_desc.B.element,
@@ -630,12 +642,13 @@ Status Conv2dOperationProfiler::initialize_workspace(
       problem_.extent_b(operation_desc.conv_kind),
       conv_workspace_.configuration.stride_b,
       conv_workspace_.problem_count,
-      seed_shift++
+      seed_shift++,
+      0 // device_index
     );
 
     if(problem_.groups == problem_.c && problem_.groups == problem_.k){
       // Depthwise direct conv kernel needs reorder the filter.
-      conv_workspace_.reordered_B = device_context.allocate_tensor(
+      conv_workspace_.reordered_B = device_context.allocate_and_initialize_tensor(
         options,
         "B",
         operation_desc.B.element,
@@ -643,11 +656,12 @@ Status Conv2dOperationProfiler::initialize_workspace(
         problem_.extent_b(operation_desc.conv_kind),
         conv_workspace_.configuration.stride_b,
         conv_workspace_.problem_count,
-        seed_shift++
+        seed_shift++,
+        0 // device_index
       );
     }
 
-    conv_workspace_.C = device_context.allocate_tensor(
+    conv_workspace_.C = device_context.allocate_and_initialize_tensor(
       options,
       "C",
       operation_desc.C.element,
@@ -655,25 +669,30 @@ Status Conv2dOperationProfiler::initialize_workspace(
       problem_.extent_c(operation_desc.conv_kind),
       conv_workspace_.configuration.stride_c,
       conv_workspace_.problem_count,
-      seed_shift++
+      seed_shift++,
+      0 // device_index
     );
 
     conv_workspace_.Computed = device_context.allocate_tensor(
+      options,
       "D",
       operation_desc.C.element,
       operation_desc.C.layout,
       problem_.extent_c(operation_desc.conv_kind),
       conv_workspace_.configuration.stride_c,
-      conv_workspace_.problem_count
+      conv_workspace_.problem_count,
+      0 // device_index
     );
 
     conv_workspace_.Reference = device_context.allocate_tensor(
+      options,
       "Reference",
       operation_desc.C.element,
       operation_desc.C.layout,
       problem_.extent_c(operation_desc.conv_kind),
       conv_workspace_.configuration.stride_c,
-      conv_workspace_.problem_count
+      conv_workspace_.problem_count,
+      0 // device_index
     );
   }
 
@@ -706,10 +725,10 @@ Status Conv2dOperationProfiler::initialize_workspace(
         conv_workspace_.reduction_host_workspace.resize(workspace_size, 0);
 
         status = reduction_op_->initialize(
-          &conv_workspace_.reduction_configuration, 
-          conv_workspace_.reduction_host_workspace.data(), 
+          &conv_workspace_.reduction_configuration,
+          conv_workspace_.reduction_host_workspace.data(),
           nullptr);
-        
+
         if (status != Status::kSuccess) {
           return status;
         }
@@ -736,7 +755,7 @@ Status Conv2dOperationProfiler::initialize_workspace(
 
 /// Verifies CUTLASS against references
 bool Conv2dOperationProfiler::verify_cutlass(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
@@ -769,7 +788,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
   }
 
   conv_workspace_.Computed->copy_from_device(conv_workspace_.C->data());
-  
+
   if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
     // update library::ConvArguments for parallel split-k reduction
     conv_workspace_.arguments.D = conv_workspace_.device_workspace.data();
@@ -799,9 +818,9 @@ bool Conv2dOperationProfiler::verify_cutlass(
   }
 
 #if 0
-  std::cout << "profiling         : " << std::endl 
-            << "conv2d            : " << operation->description().name << std::endl 
-            << "underlying conv2d : " << underlying_operation->description().name << std::endl 
+  std::cout << "profiling         : " << std::endl
+            << "conv2d            : " << operation->description().name << std::endl
+            << "underlying conv2d : " << underlying_operation->description().name << std::endl
             << "reduction         : " << reduction_op_->description().name << std::endl;
 #endif
 
@@ -818,7 +837,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
 
   // Run parallel reduction kernel for parallel split_k_mode
   if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
-    
+
     results_.back().status = reduction_op_->run(
       &conv_workspace_.reduction_arguments,
       conv_workspace_.reduction_host_workspace.data(),
@@ -840,7 +859,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
 
   // CUTLASS op ran the but not yet verified against any verification provider
   results_.back().disposition = Disposition::kNotVerified;
-  
+
   //
   // Run verification providers
   //
@@ -856,7 +875,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
 
       Status status = cudnn_satisfies(conv_desc, conv_workspace_.configuration);
 
-      // Initialize reference data to the source data 
+      // Initialize reference data to the source data
       conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
 
       if (status == Status::kSuccess) {
@@ -884,7 +903,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
     // Run verification device reference
     if (options.verification.provider_enabled(library::Provider::kReferenceDevice)) {
 
-      // Restore reference data back to initial source data 
+      // Restore reference data back to initial source data
       conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
 
       verify_with_device_reference_(
@@ -893,13 +912,13 @@ bool Conv2dOperationProfiler::verify_cutlass(
         device_context,
         operation,
         problem_space,
-        problem);      
+        problem);
     }
 
     // Run verification host reference
     if (options.verification.provider_enabled(library::Provider::kReferenceHost)) {
-      
-      // Restore reference data back to initial source data 
+
+      // Restore reference data back to initial source data
       conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
 
       verify_with_host_reference_(
@@ -908,10 +927,10 @@ bool Conv2dOperationProfiler::verify_cutlass(
         device_context,
         operation,
         problem_space,
-        problem);      
+        problem);
     }
 
-    // Update disposition to worst case verification outcome among all 
+    // Update disposition to worst case verification outcome among all
     // verification providers which are supported
     bool is_any_verification_run_passed = false;
     for(auto &m : results_.back().verification_map) {
@@ -936,7 +955,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
 
 /// Verifies CUTLASS against host reference
 bool Conv2dOperationProfiler::verify_with_host_reference_(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
@@ -954,14 +973,14 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
 
     library::ConvFunctionalKey conv2d_key(
       library::Provider::kReferenceHost,
-      conv_desc.conv_kind,        
+      conv_desc.conv_kind,
       conv_desc.A.element,
       conv_desc.A.layout,
       conv_desc.B.element,
       conv_desc.B.layout,
       conv_desc.C.element,
       conv_desc.C.layout,
-      conv_desc.tile_description.math_instruction.element_accumulator, 
+      conv_desc.tile_description.math_instruction.element_accumulator,
       conv_desc.element_epilogue);
 
 #if 0 // debug print to check which host reference instance is selected
@@ -974,12 +993,12 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
 
       results_.back().verification_map[library::Provider::kReferenceHost] = Disposition::kNotRun;
       return true;
-    }    
+    }
 
     // conv2d host reference minimum cc is 0 (CPU) and no iterator algorithm
     library::ConvPreferenceKey preference_key(0, library::IteratorAlgorithmID::kNone);
     auto cc_it = operators_it->second.find(preference_key);
-    
+
     if(cc_it == operators_it->second.end()) {
       results_.back().verification_map[library::Provider::kReferenceHost] = Disposition::kNotRun;
       return true;
@@ -1052,9 +1071,9 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
     );
 
     // Save workspace if incorrect
-    if (options.verification.save_workspace == SaveWorkspace::kIncorrect && 
+    if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
       results_.back().verification_map[library::Provider::kReferenceHost] == Disposition::kIncorrect) {
-  
+
       save_workspace(
         device_context,
         options,
@@ -1070,7 +1089,7 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
 
 /// Verifies CUTLASS against host reference
 bool Conv2dOperationProfiler::verify_with_device_reference_(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
@@ -1088,14 +1107,14 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
 
     library::ConvFunctionalKey conv2d_key(
       library::Provider::kReferenceDevice,
-      conv_desc.conv_kind,      
+      conv_desc.conv_kind,
       conv_desc.A.element,
       conv_desc.A.layout,
       conv_desc.B.element,
       conv_desc.B.layout,
       conv_desc.C.element,
       conv_desc.C.layout,
-      conv_desc.tile_description.math_instruction.element_accumulator, 
+      conv_desc.tile_description.math_instruction.element_accumulator,
       conv_desc.element_epilogue);
 
     auto operators_it = Singleton::get().operation_table.conv2d_operations.find(conv2d_key);
@@ -1105,12 +1124,12 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
       results_.back().verification_map[library::Provider::kReferenceDevice] = Disposition::kNotRun;
 
       return true;
-    }    
+    }
 
     // conv2d device reference minimum cc is 50 and no iterator algorithm
     library::ConvPreferenceKey preference_key(50, library::IteratorAlgorithmID::kNone);
     auto cc_it = operators_it->second.find(preference_key);
-    
+
     if(cc_it == operators_it->second.end()) {
       results_.back().verification_map[library::Provider::kReferenceDevice] = Disposition::kNotRun;
 
@@ -1119,7 +1138,7 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
 
     // device reference has only one instances in Conv2dOperationVectorMap
     library::Operation const *reference_op = cc_it->second[0];
-  
+
     //
     // Initialize device reference operation
     //
@@ -1166,9 +1185,9 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
     );
 
     // Save workspace if incorrect
-    if (options.verification.save_workspace == SaveWorkspace::kIncorrect && 
+    if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
       results_.back().verification_map[library::Provider::kReferenceDevice] == Disposition::kIncorrect) {
-  
+
       save_workspace(
         device_context,
         options,
@@ -1183,14 +1202,14 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
 
 /// Measures performance results
 bool Conv2dOperationProfiler::profile(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
   ProblemSpace const &problem_space,
   ProblemSpace::Problem const &problem) {
 
-  
+
   if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
 
     // Initialize structure containing Conv2d arguments
@@ -1242,7 +1261,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
   GpuTimer timer;
 
   // initialize conv2d underlying operation to handle parallel reduction
-  library::Operation const* underlying_operation = operation; 
+  library::Operation const* underlying_operation = operation;
 
   library::ConvArguments *conv_arguments = static_cast<library::ConvArguments *>(arguments);
 
@@ -1274,7 +1293,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
     conv_arguments->B = conv_workspace_.B->batch_data(problem_idx);
     conv_arguments->C = conv_workspace_.C->batch_data(problem_idx);
     conv_arguments->D = conv_workspace_.Computed->batch_data(problem_idx);
-    
+
     if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
       // update library::ConvArguments for parallel split-k reduction
       conv_arguments->D = conv_workspace_.device_workspace.data();
@@ -1304,7 +1323,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
       return status;
     }
   }
-  
+
   //
   // Initialize GPU timer
   //
@@ -1319,7 +1338,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
 
   int iteration = 0;
   for (; iteration < Iterations; ++iteration) {
-    
+
     // Setup rotating workspace
     int problem_idx = (iteration % conv_workspace_.problem_count);
 
@@ -1345,7 +1364,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
       device_workspace);
 
     // Run parallel reduction kernel for parallel split_k_mode
-    if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {      
+    if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
 
       status = reduction_op_->run(
         &conv_workspace_.reduction_arguments,
@@ -1367,7 +1386,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
   //
   // Update performance result
   //
-  
+
   runtime = timer.duration(iteration);
 
   return status;
@@ -1378,13 +1397,13 @@ Status Conv2dOperationProfiler::profile_cutlass_(
 
 /// Verifies CUTLASS against cudnn reference
 bool Conv2dOperationProfiler::verify_with_cudnn_(
-  Options const &options,  
+  Options const &options,
   PerformanceReport &report,
   DeviceContext &device_context,
   library::Operation const *operation,
   ProblemSpace const &problem_space,
   ProblemSpace::Problem const &problem) {
-  
+
   auto &conv_desc = static_cast<library::ConvDescription const &>(operation->description());
 
   //
@@ -1395,7 +1414,7 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
   cudnnStatus_t status = handle.get_cudnn_create_status();
 
   if (status != CUDNN_STATUS_SUCCESS) {
-    
+
     results_.back().verification_map[library::Provider::kCUDNN] = get_cutlass_disposition(status);
     return true;
   }
@@ -1411,7 +1430,7 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
   conv_workspace_.arguments.alpha = problem_.alpha.data();
   conv_workspace_.arguments.beta = problem_.beta.data();
   conv_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
-      
+
   // cuDNN does not support four tensor arguments, so we copy the tensor C data into
   // tensor D.
   conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
@@ -1423,8 +1442,8 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
     // Construct dispatcher to cudnn operator
     //
 
-    detail::cudnnConvDispatcher conv_op( 
-      conv_desc, 
+    detail::cudnnConvDispatcher conv_op(
+      conv_desc,
       conv_workspace_.configuration,
       conv_workspace_.arguments,
       handle
@@ -1462,7 +1481,7 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
     );
 
     // Save workspace if incorrect
-    if (options.verification.save_workspace == SaveWorkspace::kIncorrect && 
+    if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
       results_.back().verification_map[library::Provider::kCUDNN] == Disposition::kIncorrect) {
 
       save_workspace(