CUTLASS 3.5.1 (#1623)

* CUTLASS 3.5.1

* updates, optimizations, fixes

tools/profiler/src/cutlass_profiler.cu

@@ -143,16 +143,19 @@ void CutlassProfiler::enumerate_() {
 /// Profiles all operations
 int CutlassProfiler::profile_() {
 
-  int result = 0;
+  // Keep track of all device memory tensor in map
   DeviceContext device_context;
-  // For all profilers
+
+  int result = 0;
+  // For all profilers (e.g. gemm/sparse_gemm/conv2d...)
   for (auto & profiler : operation_profilers_) {
 
     if (options_.operation_kind == library::OperationKind::kInvalid ||
-      options_.operation_kind == profiler->kind()) {
+        options_.operation_kind == profiler->kind()) {
 
       result = profiler->profile_all(options_, library::Singleton::get().manifest, device_context);
 
+      // If some profile failed, terminate immediately
       if (result) {
         return result;
       }

tools/profiler/src/gemm_operation_profiler.cu

@@ -75,6 +75,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
       {ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
       {ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
       {ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},
+      {ArgumentTypeID::kInteger, {"swizzle_size", "swizzle-size"}, "Size to swizzle"},
     },
     { library::Provider::kCUBLAS}
   ) {
@@ -191,6 +192,14 @@ Status GemmOperationProfiler::GemmProblem::parse(
     this->mode = library::GemmUniversalMode::kBatched;
   }
 
+  if (!arg_as_int(this->swizzle_size, "swizzle_size", problem_space, problem)) {
+    // default value
+    this->swizzle_size = 1;
+    if (this->swizzle_size <= 0) {
+      return Status::kErrorInvalidProblem;
+    }
+  }
+
   if (!arg_as_RasterOrder(this->raster_order, "raster_order", problem_space, problem)) {
     // default value
     this->raster_order = library::RasterOrder::kHeuristic;
@@ -329,6 +338,8 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);
   set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
+  set_argument(result, "swizzle_size", problem_space, swizzle_size);
+
   set_argument(result, "alpha", problem_space,
     library::lexical_cast(alpha, operation_desc.element_epilogue));
 
@@ -383,6 +394,7 @@ Status GemmOperationProfiler::initialize_configuration(
   gemm_workspace_.arguments.alpha = problem_.alpha.data();
   gemm_workspace_.arguments.beta = problem_.beta.data();
   gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
+  gemm_workspace_.arguments.swizzle_size = problem_.swizzle_size;
   gemm_workspace_.arguments.raster_order = problem_.raster_order;
   // initialize reduction operation for parallel splitKMode
   if (problem_.split_k_mode == library::SplitKMode::kParallel) {

tools/profiler/src/operation_profiler.cu

@@ -345,6 +345,7 @@ int OperationProfiler::profile_all(
 
     // For each operation in manifest
     int matched_operation_count = 0;
+    int profiled_operation_count = 0;
     for (auto const& operation_ptr : manifest) {
 
       library::Operation const *operation = operation_ptr.get();
@@ -434,7 +435,7 @@ int OperationProfiler::profile_all(
           // If there was an internal error, consume the CUDA error and move to the next operation.
           (void)cudaGetLastError();
 
-          report.append_results(results_);
+          report.append_result(model_result_);
           continue;
         }
         else if (status != Status::kSuccess) {
@@ -522,25 +523,42 @@ int OperationProfiler::profile_all(
             operation,
             problem_space,
             problem);
+
+          // Count op as profiled, even it failed to profile
+          profiled_operation_count++;
         }
 
         report.append_results(results_);
         results_.clear();
-      }
+      } // if op satisfied compute capacity
 
       if (!continue_profiling) {
+        // break out of `for op in manifest` loop and move to next problem
+        // `for each problem in problem space` conditional check on not continue profiling
         break;
       }
-    }
+    } // for op in manifest
 
     // If we did not find any kernels that match our filters and error_on_no_match was set, report an error
     if (options.profiling.error_on_no_match && matched_operation_count <= 0) {
       #if !NDEBUG
-      std::cout << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
+      std::cerr << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
       #endif
-      retval = 1;
+      retval |= 1;
+      // Stop profiling on error no match
+      continue_profiling = false;
     }
-  }
+
+    if (options.profiling.error_if_nothing_is_profiled && options.profiling.enabled && profiled_operation_count <= 0) {
+      #if !NDEBUG
+      std::cerr << "Error: No kernels profiled found with kernel selection filters [--error_if_nothing_is_profiled]" << std::endl;
+      #endif
+      retval |= 1;
+      // Stop profiling on error no match
+      continue_profiling = false;
+    }
+
+  } // for each problem in problem space
 
   return retval;
 }

tools/profiler/src/options.cu

@@ -726,11 +726,13 @@ Options::Options(cutlass::CommandLine const &cmdline):
   else if (cmdline.check_cmd_line_flag("kernels")) {
     cmdline.get_cmd_line_arguments("kernels", operation_names);
     profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
+    profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
   }
 
   if (cmdline.check_cmd_line_flag("ignore-kernels")) {
     cmdline.get_cmd_line_arguments("ignore-kernels", excluded_operation_names);
     profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
+    profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
   }
 
   // Prevent launches on the device for anything other than CUTLASS operation

tools/profiler/src/problem_space.cpp

@@ -395,10 +395,6 @@ std::unique_ptr<KernelArgument::ValueIterator> EnumeratedTypeArgument::end() con
 
 //////////////////////////////////////////////////////////////////////////////////////////////////
 
-ProblemSpace::Iterator::Iterator() {
-
-}
-
 ProblemSpace::Iterator::Iterator(ProblemSpace const &problem_space) {
   for (auto const & arg_ptr : problem_space.arguments) {
     construct_(arg_ptr.get());