CUTLASS 3.5.1 (#1623)

* CUTLASS 3.5.1

* updates, optimizations, fixes

tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h

@@ -84,6 +84,8 @@ public:
     int batch_count{1};
 
     cutlass::library::RasterOrder raster_order{cutlass::library::RasterOrder::kHeuristic};
+    int swizzle_size{1};
+
     // gemm with parallel interleaved reduction
     // gemm epilogue (alpha, beta) = (1.0, 0.0)
     // reduction epilogue (alpha, beta) = (GemmProblem::alpha, GemmProblem::beta)

tools/profiler/include/cutlass/profiler/options.h

@@ -72,7 +72,7 @@ public:
     // Methods
     //
 
-    Library(CommandLine const &cmdline);
+    explicit Library(CommandLine const &cmdline);
 
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -94,7 +94,7 @@ public:
     // Methods
     //
 
-    Device(CommandLine const &cmdline);
+    explicit Device(CommandLine const &cmdline);
 
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -128,7 +128,7 @@ public:
     // Methods
     //
 
-    Initialization(CommandLine const &cmdline);
+    explicit Initialization(CommandLine const &cmdline);
     
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -170,7 +170,7 @@ public:
     // Methods
     //
 
-    Verification(CommandLine const &cmdline);
+    explicit Verification(CommandLine const &cmdline);
   
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -186,22 +186,26 @@ public:
   struct Profiling {
 
     /// Number of workspaces to rotate through to avoid cache-resident working sets
-    int workspace_count;
+    int workspace_count{0};
 
     /// Number of iterations to warmup each kernel prior to profiling
-    int warmup_iterations;
+    int warmup_iterations{10};
 
     /// Number of iterations to profile each kernel - if 0, kernels are launched up to the profiling duration
-    int iterations;
+    int iterations{100};
 
     /// Number of ms to sleep between profiling periods (ms)
-    int sleep_duration;
+    int sleep_duration{50};
 
     /// If true, profiling is actually conducted.
-    bool enabled;
+    bool enabled{true};
 
     /// If true, profiling returns an error code if no kernels are found to match the filters.
-    bool error_on_no_match = false;
+    bool error_on_no_match{false};
+
+    /// If true, profiling returns an error code if no kernel are profiled
+    // Sometimes the kernel matches but failed to profile (e.g. can_implement() error)
+    bool error_if_nothing_is_profiled{false};
 
     /// List of providers of each functionality to be profiled
     ProviderVector providers;
@@ -210,7 +214,7 @@ public:
     // Methods
     //
 
-    Profiling(CommandLine const &cmdline);
+    explicit Profiling(CommandLine const &cmdline);
 
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -255,7 +259,7 @@ public:
     // Methods
     //
 
-    Report(CommandLine const &cmdline);
+    explicit Report(CommandLine const &cmdline);
     
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -277,7 +281,7 @@ public:
     // Methods
     //
 
-    About(CommandLine const &cmdline);
+    explicit About(CommandLine const &cmdline);
     
     void print_usage(std::ostream &out) const;
     void print_options(std::ostream &out, int indent = 0) const;
@@ -320,7 +324,7 @@ public:
 
 public:
 
-  Options(CommandLine const &cmdline);
+  explicit Options(CommandLine const &cmdline);
 
   void print_usage(std::ostream &out) const;
   void print_options(std::ostream &out) const;

tools/profiler/include/cutlass/profiler/problem_space.h

@@ -256,7 +256,7 @@ struct ScalarArgument : public KernelArgument {
     virtual std::ostream &print(std::ostream &out) const;
   };
 
-  using ValueCollection = std::vector<std::string>;
+  using ValueCollection = std::vector<decltype(ScalarValue::value)>;
 
   /// Abstract base class to iterate over values within arguments
   struct ScalarValueIterator : public KernelArgument::ValueIterator {
@@ -271,7 +271,7 @@ struct ScalarArgument : public KernelArgument {
     // Methods
     //
 
-    ScalarValueIterator(ScalarArgument const *argument = nullptr);
+    explicit ScalarValueIterator(ScalarArgument const *argument = nullptr);
 
     virtual void operator++();
     virtual bool operator==(ValueIterator const &it) const;
@@ -292,7 +292,7 @@ struct ScalarArgument : public KernelArgument {
   //
 
   /// Default ctor
-  ScalarArgument(
+  explicit ScalarArgument(
     ArgumentDescription const *description
   ): 
     KernelArgument(description) { }
@@ -632,7 +632,7 @@ struct TensorArgument : public KernelArgument {
     // Methods
     //
 
-    TensorValueIterator(TensorArgument const *argument_);
+    explicit TensorValueIterator(TensorArgument const *argument_);
 
     virtual void operator++();
     virtual bool operator==(ValueIterator const &it) const;
@@ -649,7 +649,7 @@ struct TensorArgument : public KernelArgument {
   //
 
   /// Default ctor
-  TensorArgument(
+  explicit TensorArgument(
     ArgumentDescription const *description
   ): 
     KernelArgument(description) { }
@@ -690,7 +690,7 @@ struct EnumeratedTypeArgument : public KernelArgument {
     virtual std::ostream &print(std::ostream &out) const;
   };
 
-  using ValueCollection = std::vector<std::string>;
+  using ValueCollection = std::vector<decltype(EnumeratedTypeValue::element)>;
 
   /// Abstract base class to iterate over values within arguments
   struct EnumeratedTypeValueIterator : public KernelArgument::ValueIterator {
@@ -705,7 +705,7 @@ struct EnumeratedTypeArgument : public KernelArgument {
     // Methods
     //
 
-    EnumeratedTypeValueIterator(EnumeratedTypeArgument const *argument_ = nullptr);
+    explicit EnumeratedTypeValueIterator(EnumeratedTypeArgument const *argument_ = nullptr);
 
     virtual void operator++();
     virtual bool operator==(ValueIterator const &it) const;
@@ -725,7 +725,7 @@ struct EnumeratedTypeArgument : public KernelArgument {
   //
 
   /// Default ctor
-  EnumeratedTypeArgument(ArgumentDescription const *description):
+  explicit EnumeratedTypeArgument(ArgumentDescription const *description):
     KernelArgument(description) {}
 
   virtual bool not_null() const {
@@ -819,7 +819,7 @@ public:
   //
 
   /// Default ctor
-  ProblemSpace() {}
+  ProblemSpace() = default;
 
   /// Constructs a problem space from a vector of arguments. This vector must outlive
   /// the ProblemSpace object, which stores pointers to objects within the

tools/profiler/src/cutlass_profiler.cu

@@ -143,16 +143,19 @@ void CutlassProfiler::enumerate_() {
 /// Profiles all operations
 int CutlassProfiler::profile_() {
 
-  int result = 0;
+  // Keep track of all device memory tensor in map
   DeviceContext device_context;
-  // For all profilers
+
+  int result = 0;
+  // For all profilers (e.g. gemm/sparse_gemm/conv2d...)
   for (auto & profiler : operation_profilers_) {
 
     if (options_.operation_kind == library::OperationKind::kInvalid ||
-      options_.operation_kind == profiler->kind()) {
+        options_.operation_kind == profiler->kind()) {
 
       result = profiler->profile_all(options_, library::Singleton::get().manifest, device_context);
 
+      // If some profile failed, terminate immediately
       if (result) {
         return result;
       }

tools/profiler/src/gemm_operation_profiler.cu

@@ -75,6 +75,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
       {ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
       {ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
       {ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},
+      {ArgumentTypeID::kInteger, {"swizzle_size", "swizzle-size"}, "Size to swizzle"},
     },
     { library::Provider::kCUBLAS}
   ) {
@@ -191,6 +192,14 @@ Status GemmOperationProfiler::GemmProblem::parse(
     this->mode = library::GemmUniversalMode::kBatched;
   }
 
+  if (!arg_as_int(this->swizzle_size, "swizzle_size", problem_space, problem)) {
+    // default value
+    this->swizzle_size = 1;
+    if (this->swizzle_size <= 0) {
+      return Status::kErrorInvalidProblem;
+    }
+  }
+
   if (!arg_as_RasterOrder(this->raster_order, "raster_order", problem_space, problem)) {
     // default value
     this->raster_order = library::RasterOrder::kHeuristic;
@@ -329,6 +338,8 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
   set_argument(result, "split_k_slices", problem_space, split_k_slices);
   set_argument(result, "batch_count", problem_space, batch_count);
   set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
+  set_argument(result, "swizzle_size", problem_space, swizzle_size);
+
   set_argument(result, "alpha", problem_space,
     library::lexical_cast(alpha, operation_desc.element_epilogue));
 
@@ -383,6 +394,7 @@ Status GemmOperationProfiler::initialize_configuration(
   gemm_workspace_.arguments.alpha = problem_.alpha.data();
   gemm_workspace_.arguments.beta = problem_.beta.data();
   gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
+  gemm_workspace_.arguments.swizzle_size = problem_.swizzle_size;
   gemm_workspace_.arguments.raster_order = problem_.raster_order;
   // initialize reduction operation for parallel splitKMode
   if (problem_.split_k_mode == library::SplitKMode::kParallel) {

tools/profiler/src/operation_profiler.cu

@@ -345,6 +345,7 @@ int OperationProfiler::profile_all(
 
     // For each operation in manifest
     int matched_operation_count = 0;
+    int profiled_operation_count = 0;
     for (auto const& operation_ptr : manifest) {
 
       library::Operation const *operation = operation_ptr.get();
@@ -434,7 +435,7 @@ int OperationProfiler::profile_all(
           // If there was an internal error, consume the CUDA error and move to the next operation.
           (void)cudaGetLastError();
 
-          report.append_results(results_);
+          report.append_result(model_result_);
           continue;
         }
         else if (status != Status::kSuccess) {
@@ -522,25 +523,42 @@ int OperationProfiler::profile_all(
             operation,
             problem_space,
             problem);
+
+          // Count op as profiled, even it failed to profile
+          profiled_operation_count++;
         }
 
         report.append_results(results_);
         results_.clear();
-      }
+      } // if op satisfied compute capacity
 
       if (!continue_profiling) {
+        // break out of `for op in manifest` loop and move to next problem
+        // `for each problem in problem space` conditional check on not continue profiling
         break;
       }
-    }
+    } // for op in manifest
 
     // If we did not find any kernels that match our filters and error_on_no_match was set, report an error
     if (options.profiling.error_on_no_match && matched_operation_count <= 0) {
       #if !NDEBUG
-      std::cout << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
+      std::cerr << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
       #endif
-      retval = 1;
+      retval |= 1;
+      // Stop profiling on error no match
+      continue_profiling = false;
     }
-  }
+
+    if (options.profiling.error_if_nothing_is_profiled && options.profiling.enabled && profiled_operation_count <= 0) {
+      #if !NDEBUG
+      std::cerr << "Error: No kernels profiled found with kernel selection filters [--error_if_nothing_is_profiled]" << std::endl;
+      #endif
+      retval |= 1;
+      // Stop profiling on error no match
+      continue_profiling = false;
+    }
+
+  } // for each problem in problem space
 
   return retval;
 }

tools/profiler/src/options.cu

@@ -726,11 +726,13 @@ Options::Options(cutlass::CommandLine const &cmdline):
   else if (cmdline.check_cmd_line_flag("kernels")) {
     cmdline.get_cmd_line_arguments("kernels", operation_names);
     profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
+    profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
   }
 
   if (cmdline.check_cmd_line_flag("ignore-kernels")) {
     cmdline.get_cmd_line_arguments("ignore-kernels", excluded_operation_names);
     profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
+    profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
   }
 
   // Prevent launches on the device for anything other than CUTLASS operation

tools/profiler/src/problem_space.cpp

@@ -395,10 +395,6 @@ std::unique_ptr<KernelArgument::ValueIterator> EnumeratedTypeArgument::end() con
 
 //////////////////////////////////////////////////////////////////////////////////////////////////
 
-ProblemSpace::Iterator::Iterator() {
-
-}
-
 ProblemSpace::Iterator::Iterator(ProblemSpace const &problem_space) {
   for (auto const & arg_ptr : problem_space.arguments) {
     construct_(arg_ptr.get());