CUTLASS 3.4.0 (#1286)

* CUTLASS 3.4.0 * Update CHANGELOG.md --------- Co-authored-by: Pradeep Ramani <prramani@nvidia.com>
2023-12-29 12:21:31 -08:00
parent b7508e3379
commit 8236f30675
211 changed files with 11409 additions and 2763 deletions
--- a/tools/library/CMakeLists.txt
+++ b/tools/library/CMakeLists.txt
@@ -136,6 +136,8 @@ function(cutlass_add_cutlass_library)
      EXPORT_NAME ${__EXPORT_NAME}
      ""
      )
+
+    target_compile_features(${__NAME} INTERFACE cxx_std_17)
    
    set_target_properties(
      ${__NAME}
@@ -159,6 +161,8 @@ function(cutlass_add_cutlass_library)
      EXPORT_NAME ${__EXPORT_NAME}_static
      ""
      )
+
+    target_compile_features(${__NAME}_static INTERFACE cxx_std_17)
    
    if (WIN32)
      set(STATIC_OUTPUT_NAME ${__OUTPUT_NAME}.static)
@@ -196,8 +200,8 @@ function(cutlass_add_cutlass_library)
      # to the main cutlass library so users automatically get the necessary link
      # commands to pull in all kernels by default.
    
-      target_link_libraries(${DEFAULT_NAME} INTERFACE ${__NAME})
-      target_link_libraries(${DEFAULT_NAME}_static INTERFACE ${__NAME}_static)
+      target_link_libraries(${DEFAULT_NAME} PUBLIC ${__NAME})
+      target_link_libraries(${DEFAULT_NAME}_static PUBLIC ${__NAME}_static)
    
    endif()

@@ -246,6 +250,7 @@ cutlass_add_cutlass_library(

 # For backward compatibility with the old name
 add_library(cutlass_lib ALIAS cutlass_library)
+add_library(cutlass_lib_static ALIAS cutlass_library_static)

 ################################################################################

--- a/tools/library/include/cutlass/library/handle.h
+++ b/tools/library/include/cutlass/library/handle.h
@@ -65,7 +65,7 @@ private:

  /// Size of device workspace in bytes
  size_t workspace_size_;
-    
+
  /// Indicates whether scalars are host or device pointers
  ScalarPointerMode scalar_pointer_mode_;

@@ -89,7 +89,7 @@ public:
  //
  // Persistent state accessors
  //
-  
+
  /// Returns compute capability of the selected device
  int compute_capability() const;

@@ -135,7 +135,7 @@ public:
    int K,                                    /// GEMM K dimension

    NumericTypeID element_compute,            /// Data type of internal accumulation
-    
+
    NumericTypeID element_scalar,             /// Data type of alpha/beta scalars

    void const *alpha,                        /// Pointer to alpha scalar
@@ -164,7 +164,7 @@ public:
    void * ptr_D,                             /// Pointer to D matrix
    int64_t ldd                               /// Leading dimension of D matrix
  );
-  
+
  /// Executes a GEMM computation: D <= alpha * A*B + beta * C.
  //
  // Supports batched-strided, batched array or split-K serial or split-K parallel.
@@ -176,7 +176,6 @@ public:
    int M,                                    /// GEMM M dimension
    int N,                                    /// GEMM N dimension
    int K,                                    /// GEMM K dimension
-
    NumericTypeID element_compute,            /// Data type of internal accumulation

    NumericTypeID element_scalar,             /// Data type of alpha/beta scalars
@@ -218,7 +217,7 @@ public:
  /// Planar complex GEMM
  ///
  /// Note, all data types are the real-valued base types used by the planar-complex GEMM kernel.
-  ///                       
+  ///
  Status gemm_planar_complex(

    int M,                                    /// GEMM M dimension
@@ -245,7 +244,7 @@ public:
    ComplexTransform transform_B,             /// Complex transformation applied to B matrix

    void const * ptr_B_real,                  /// Pointer to real part of B matrix
-    void const * ptr_B_imag,                  /// Pointer to imaginary part of B matrix 
+    void const * ptr_B_imag,                  /// Pointer to imaginary part of B matrix
    int64_t ldb_real,                         /// Leading dimension of real part of B matrix
    int64_t ldb_imag,                         /// Leading dimension of imaginary part of B matrix

@@ -301,7 +300,7 @@ public:
    ComplexTransform transform_A,             /// Complex transformation applied to A matrix

    void const * const * ptr_A_real,          /// Pointer to array containing pointers to real part of A matrices
-    void const * const * ptr_A_imag,          /// Pointer to array containing pointers to imaginary part of A matrices 
+    void const * const * ptr_A_imag,          /// Pointer to array containing pointers to imaginary part of A matrices

    int64_t lda_real,                         /// Leading dimension of real part of A matrix
    int64_t lda_imag,                         /// Leading dimension of imaginary part of A matrix
--- a/tools/library/include/cutlass/library/library.h
+++ b/tools/library/include/cutlass/library/library.h
@@ -28,17 +28,17 @@
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 **************************************************************************************************/
-/*! 
+/*!
  \file

  \brief CUTLASS Library is an object-oriented approach to managing operations implemented by CUTLASS.

  Generally,
-    
+
    description   - compile-time constant parameters used to instantiate an operation

-    configuration - runtime parameters with computationally expensive initialization 
-    
+    configuration - runtime parameters with computationally expensive initialization
+
    arguments     - runtime parameters that may be passed to an initialized operation with low
                    computational overhead
 */
@@ -87,26 +87,26 @@ public:
  virtual OperationDescription const & description() const = 0;

  virtual Status can_implement(
-    void const *configuration, 
+    void const *configuration,
    void const *arguments) const = 0;
-  
+
  virtual uint64_t get_host_workspace_size(
    void const *configuration) const = 0;
-  
+
  virtual uint64_t get_device_workspace_size(
    void const *configuration,
    void const *arguments = nullptr) const = 0;
-  
+
  virtual Status initialize(
-    void const *configuration, 
-    void *host_workspace, 
-    void *device_workspace = nullptr, 
+    void const *configuration,
+    void *host_workspace,
+    void *device_workspace = nullptr,
    cudaStream_t stream = nullptr) const = 0;

  virtual Status run(
    void const *arguments,
-    void *host_workspace, 
-    void *device_workspace = nullptr, 
+    void *host_workspace,
+    void *device_workspace = nullptr,
    cudaStream_t stream = nullptr) const = 0;

 };
@@ -217,7 +217,7 @@ using GemmBatchedArguments = GemmArguments;
 struct GemmArrayConfiguration {

  gemm::GemmCoord problem_size;
-  
+
  /// Leading dimension of A matrix
  int64_t lda;

@@ -241,7 +241,7 @@ struct GemmArrayArguments {
  void * const *D;
  void const *alpha;
  void const *beta;
-  ScalarPointerMode pointer_mode;  
+  ScalarPointerMode pointer_mode;
 };

 /////////////////////////////////////////////////////////////////////////////////////////////////
@@ -264,7 +264,7 @@ struct GemmUniversalConfiguration {
 };

 struct GemmUniversalArguments {
-  // NOTE: these are replicated for 3.0 interfaces 
+  // NOTE: these are replicated for 3.0 interfaces
  gemm::GemmCoord problem_size;
  int batch_count;

@@ -645,8 +645,8 @@ struct SymmArguments {
 struct Conv2dConfiguration {

  conv::SplitKMode split_k_mode;
-  
-  /// Conv2d problem size 
+
+  /// Conv2d problem size
  //  contains strictly conv2d size (N,H,W,C,K,R,S,P,Q,padding,stride,dilation,mode)
  //  also includes (split_k_slices, groups)
  conv::Conv2dProblemSize problem_size;
@@ -669,8 +669,8 @@ struct Conv2dConfiguration {
 struct Conv3dConfiguration {

  conv::SplitKMode split_k_mode;
-  
-  /// Conv2d problem size 
+
+  /// Conv2d problem size
  //  contains strictly conv2d size (N,D,H,W,C,K,T,R,S,Z,P,Q,padding,stride,dilation,mode)
  //  also includes (split_k_slices, groups)
  conv::Conv3dProblemSize problem_size;
@@ -688,7 +688,7 @@ struct Conv3dConfiguration {
  layout::TensorNDHWC layout_output;

  //
-  // Methods 
+  // Methods
  //

  // Mapping functions (A,B,C -> activation,filter,output)
@@ -734,7 +734,7 @@ struct ConvArguments {

  /// pointer to reordered matrix B
  void const *reordered_B;
-  
+
  /// pointer to implicit gemm matrix C
  void const *C;

@@ -770,7 +770,7 @@ struct ReductionConfiguration {
  int64_t partition_stride;

  /// leading dimension of 'w'orkspace operand
-  int64_t ldw; 
+  int64_t ldw;

  /// leading dimension of 's'ource operand
  int64_t lds;
--- a/tools/library/include/cutlass/library/manifest.h
+++ b/tools/library/include/cutlass/library/manifest.h
@@ -90,7 +90,11 @@ public:
  Status release();

  /// Appends an operation and takes ownership
-  void append(Operation *operation_ptr);
+  void append(Operation *operation_ptr) {\
+    // This function is inline s.t. it is present in generated libraries
+    // without having to compile or link in manifest.cpp
+    operations_.emplace_back(operation_ptr);
+  }

  /// Returns an iterator to the first operation
  OperationVector const &operations() const;
--- a/tools/library/src/gemm_operation_3x.hpp
+++ b/tools/library/src/gemm_operation_3x.hpp
@@ -257,11 +257,11 @@ protected:
        case RasterOrder::kAlongM:
          operator_args.scheduler.raster_order = Enum_t::AlongM;
          break;
-        default: 
+        default:
          operator_args.scheduler.raster_order = Enum_t::Heuristic;
      }
    }
-
+    
    return status;
  }

@@ -271,7 +271,7 @@ public:
  Status can_implement(
      void const *configuration_ptr, void const *arguments_ptr) const override {

-    GemmUniversalConfiguration const *configuration = 
+    GemmUniversalConfiguration const *configuration =
      static_cast<GemmUniversalConfiguration const *>(configuration_ptr);
    GemmUniversalArguments const *arguments =
      static_cast<GemmUniversalArguments const *>(arguments_ptr);
--- a/tools/library/src/handle.cu
+++ b/tools/library/src/handle.cu
@@ -32,7 +32,7 @@
 /*! \file
    \brief CUTLASS Library handle.
 */
-#include <iostream> 
+#include <iostream>
 #include <stdexcept>
 #include <cstdint>

@@ -47,14 +47,14 @@ namespace library {

 /// Constructor
 Handle::Handle(
-  cudaStream_t stream, 
+  cudaStream_t stream,
  size_t workspace_size
 ):
-  provider_(Provider::kCUTLASS), 
-  stream_(stream), 
-  workspace_(nullptr), 
-  workspace_size_(0), 
-  scalar_pointer_mode_(ScalarPointerMode::kHost), 
+  provider_(Provider::kCUTLASS),
+  stream_(stream),
+  workspace_(nullptr),
+  workspace_size_(0),
+  scalar_pointer_mode_(ScalarPointerMode::kHost),
  last_operation_(nullptr) {

  int device_idx = -1;
@@ -94,7 +94,7 @@ Handle::Handle(Handle && handle) {
  workspace_ = handle.workspace_;
  stream_ = handle.stream_;
  scalar_pointer_mode_ = handle.scalar_pointer_mode_;
-  
+
  handle.workspace_ = nullptr;
  handle.workspace_size_ = 0;
 }
@@ -156,14 +156,14 @@ void Handle::set_workspace_size(size_t bytes) {
    if (workspace_) {
      cudaFree(workspace_);
    }
-      
+
    workspace_ = nullptr;
    workspace_size_ = bytes;

    if (workspace_size_) {
-  
+
      cudaError_t error = cudaMalloc((void **)&workspace_, workspace_size_);
-  
+
      if (error != cudaSuccess) {
        throw std::runtime_error("Failed to allocate workspace");
      }
@@ -239,7 +239,7 @@ static int gemm_problem_alignment(
  };

  for (; max_alignment_in_bytes > 0; max_alignment_in_bytes /= 2) {
-    
+
    bool satisfied = true;

    // Can pointers satisfy this?
@@ -260,7 +260,7 @@ static int gemm_problem_alignment(
    int max_element_alignment = 0;

    for (NumericTypeID type_id : elements) {
-      int element_alignment = max_alignment_in_bytes * 8 / library::sizeof_bits(type_id); 
+      int element_alignment = max_alignment_in_bytes * 8 / library::sizeof_bits(type_id);
      max_element_alignment = std::max(max_element_alignment, element_alignment);
    }

@@ -286,7 +286,7 @@ static int gemm_problem_alignment(

 /// Find the best kernel in descending order of preference.
 static Operation const * find_gemm_operation(
-  GemmOperationFunctionalMap::const_iterator operators_it, 
+  GemmOperationFunctionalMap::const_iterator operators_it,
  GemmPreferenceKey const preference_key) {

  auto cc_it = operators_it->second.upper_bound(preference_key);
@@ -363,7 +363,7 @@ Status Handle::gemm(
  void * ptr_D,                             /// Pointer to D matrix
  int64_t ldd                               /// Leading dimension of D matrix
 ) {
-  
+
  //
  // Find the operation
  //
@@ -390,7 +390,7 @@ Status Handle::gemm(
  if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
    return cutlass::Status::kErrorNotSupported;
  }
-  
+
  if (operators_it->second.empty()) {
    return cutlass::Status::kErrorNotSupported;
  }
@@ -403,7 +403,7 @@ Status Handle::gemm(
  int const kMaximumAlignmentSize = 16;

  int alignment = gemm_problem_alignment(
-    M, N, K, 
+    M, N, K,
    element_A, ptr_A, lda, 0,
    element_B, ptr_B, ldb, 0,
    element_C, ptr_C, ldc, 0,
@@ -491,7 +491,6 @@ Status Handle::gemm_universal(
  int M,                                    /// GEMM M dimension
  int N,                                    /// GEMM N dimension
  int K,                                    /// GEMM K dimension
-
  NumericTypeID element_compute,            /// Data type of internal accumulation

  NumericTypeID element_scalar,             /// Data type of alpha/beta scalars
@@ -529,7 +528,7 @@ Status Handle::gemm_universal(
  int64_t batch_stride_C,                   /// Batch stride of C operand
  int64_t batch_stride_D                    /// Batch stride of D operand
 ) {
-  
+
  //
  // Find the operation
  //
@@ -556,7 +555,7 @@ Status Handle::gemm_universal(
  if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
    return cutlass::Status::kErrorNotSupported;
  }
-  
+
  if (operators_it->second.empty()) {
    return cutlass::Status::kErrorNotSupported;
  }
@@ -576,14 +575,14 @@ Status Handle::gemm_universal(
  // Ignore alignment of pointers to pointers. We can't check this from the host,
  // as each batch index has its own pointer in device memory.
  if (mode == GemmUniversalMode::kArray) {
-    ptr_A_check = nullptr; 
-    ptr_B_check = nullptr; 
-    ptr_C_check = nullptr; 
-    ptr_D_check = nullptr; 
+    ptr_A_check = nullptr;
+    ptr_B_check = nullptr;
+    ptr_C_check = nullptr;
+    ptr_D_check = nullptr;
  }

  int alignment = gemm_problem_alignment(
-    M, N, K, 
+    M, N, K,
    element_A, ptr_A_check, lda, 0,
    element_B, ptr_B_check, ldb, 0,
    element_C, ptr_C_check, ldc, 0,
@@ -758,7 +757,7 @@ Status Handle::gemm_planar_complex(
  if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
    return cutlass::Status::kErrorNotSupported;
  }
-  
+
  if (operators_it->second.empty()) {
    return cutlass::Status::kErrorNotSupported;
  }
@@ -772,14 +771,14 @@ Status Handle::gemm_planar_complex(

  int alignment = std::max(
    gemm_problem_alignment(
-      M, N, K, 
+      M, N, K,
      element_A, ptr_A_real, lda_real, batch_stride_A_real,
      element_B, ptr_B_real, ldb_real, batch_stride_B_real,
      element_C, ptr_C_real, ldc_real, batch_stride_C_real,
      ptr_D_real, ldd_real, batch_stride_D_real, kMaximumAlignmentSize
    ),
    gemm_problem_alignment(
-      M, N, K, 
+      M, N, K,
      element_A, ptr_A_imag, lda_imag, batch_stride_A_imag,
      element_B, ptr_B_imag, ldb_imag, batch_stride_B_imag,
      element_C, ptr_C_imag, ldc_imag, batch_stride_C_imag,
@@ -928,7 +927,7 @@ Status Handle::gemm_planar_complex_array(
  int64_t ldd_real,                             /// Leading dimension of real part of D matrix
  int64_t ldd_imag                              /// Leading dimension of imaginary part of D matrix
 ) {
-  
+
  //
  // Find the operation
  //
@@ -955,7 +954,7 @@ Status Handle::gemm_planar_complex_array(
  if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
    return cutlass::Status::kErrorNotSupported;
  }
-  
+
  if (operators_it->second.empty()) {
    return cutlass::Status::kErrorNotSupported;
  }
@@ -969,14 +968,14 @@ Status Handle::gemm_planar_complex_array(

  int alignment = std::max(
    gemm_problem_alignment(
-      expected_M, expected_N, expected_K, 
+      expected_M, expected_N, expected_K,
      element_A, nullptr, lda_real, 0,
      element_B, nullptr, ldb_real, 0,
      element_C, nullptr, ldc_real, 0,
      nullptr, ldd_real, 0, kMaximumAlignmentSize
    ),
    gemm_problem_alignment(
-      expected_M, expected_N, expected_K, 
+      expected_M, expected_N, expected_K,
      element_A, nullptr, lda_imag, 0,
      element_B, nullptr, ldb_imag, 0,
      element_C, nullptr, ldc_imag, 0,
@@ -1066,7 +1065,7 @@ Status Handle::gemm_planar_complex_array(
 /// Finds conv operation instances with Conv::ElementC = Reduction::ElementWorkspace
 Operation const* find_conv_operation_for_parallel_reduction(Operation const *operation) {

-  ConvDescription const &conv_desc = 
+  ConvDescription const &conv_desc =
    static_cast<ConvDescription const &>(operation->description());

  // if the curren conv operation accumulator and output data type match return operation
@@ -1077,19 +1076,19 @@ Operation const* find_conv_operation_for_parallel_reduction(Operation const *ope
  // find conv operation to match conv output and reduction workspace data type
  ConvFunctionalKey key(
    library::Provider::kCUTLASS,
-    conv_desc.conv_kind,        
+    conv_desc.conv_kind,
    conv_desc.A.element,
    conv_desc.A.layout,
    conv_desc.B.element,
    conv_desc.B.layout,
    conv_desc.tile_description.math_instruction.element_accumulator,
    conv_desc.C.layout,
-    conv_desc.tile_description.math_instruction.element_accumulator, 
+    conv_desc.tile_description.math_instruction.element_accumulator,
    conv_desc.element_epilogue);

  // conv operation table for conv2d or conv3d
-  auto conv_operations = (conv_desc.kind == OperationKind::kConv2d) ? 
-                          Singleton::get().operation_table.conv2d_operations : 
+  auto conv_operations = (conv_desc.kind == OperationKind::kConv2d) ?
+                          Singleton::get().operation_table.conv2d_operations :
                          Singleton::get().operation_table.conv3d_operations;

  // find ConvFunctionalKey in convolution operation table
@@ -1098,18 +1097,18 @@ Operation const* find_conv_operation_for_parallel_reduction(Operation const *ope
  if (operators_it == conv_operations.end()) {
    return nullptr;
  }
-  
+
  if (operators_it->second.empty()) {
    return nullptr;
  }

  // conv operation for same compute capability and iterator algorithm
  ConvPreferenceKey preference_key(
-    conv_desc.tile_description.minimum_compute_capability, 
+    conv_desc.tile_description.minimum_compute_capability,
    conv_desc.iterator_algorithm);

  auto it = operators_it->second.find(preference_key);
-  
+
  if(it == operators_it->second.end()) {
    return nullptr;
  }
@@ -1129,7 +1128,7 @@ Operation const* find_conv_operation_for_parallel_reduction(Operation const *ope
 /// Finds gemm operation instances with Gemm::ElementC = Reduction::ElementWorkspace
 Operation const* find_gemm_operation_for_parallel_reduction(Operation const *operation) {

-  GemmDescription const &gemm_desc = 
+  GemmDescription const &gemm_desc =
    static_cast<GemmDescription const &>(operation->description());

  // if the curren gemm operation accumulator and output data type match return operation
@@ -1174,7 +1173,7 @@ Operation const* find_gemm_operation_for_parallel_reduction(Operation const *ope
    gemm_desc.B.alignment);

  GemmPreferenceKey preference_key(
-    gemm_desc.tile_description.minimum_compute_capability, 
+    gemm_desc.tile_description.minimum_compute_capability,
    alignment);

  auto it = operators_it->second.find(preference_key);
--- a/tools/library/src/manifest.cpp
+++ b/tools/library/src/manifest.cpp
@@ -77,11 +77,6 @@ Status Manifest::release() {
  return Status::kSuccess;
 }

-/// Appends an operation and takes ownership
-void Manifest::append(Operation *operation_ptr) {
-  operations_.emplace_back(operation_ptr);
-}
-
 /// Returns an iterator to the first operation
 OperationVector const & Manifest::operations() const {
  return operations_;
--- a/tools/library/src/reference/conv_reference_operation.h
+++ b/tools/library/src/reference/conv_reference_operation.h
@@ -45,6 +45,7 @@
 #include "cutlass/library/util.h"
 #include "library_internal.h"

+#include "cutlass/conv/convolution.h"
 #include "cutlass/util/reference/host/convolution.h"
 #include "cutlass/util/reference/device/convolution.h"

@@ -59,7 +60,7 @@ namespace detail {

 template <
  Provider kProvider,
-  conv::Operator ConvolutionalOperator,
+  cutlass::conv::Operator ConvolutionalOperator,
  int ConvDim,
  typename ElementA_,
  typename LayoutA_,
@@ -77,7 +78,7 @@ struct ConvReferenceDispatcher;
 /// Dispatcher for Conv2d (partially specialized for kConvDim == 2)
 template <
  Provider kProvider,
-  conv::Operator kConvolutionalOperator,
+  cutlass::conv::Operator kConvolutionalOperator,
  typename ElementA,
  typename LayoutA,
  typename ElementB,
@@ -193,7 +194,7 @@ struct ConvReferenceDispatcher<
 /// Dispatcher for Conv3d (partially specialized for kConvDim == 3)
 template <
  Provider kProvider,
-  conv::Operator kConvolutionalOperator,
+  cutlass::conv::Operator kConvolutionalOperator,
  typename ElementA,
  typename LayoutA,
  typename ElementB,
@@ -292,7 +293,7 @@ struct ConvReferenceDispatcher<

 template <
  Provider Provider_,
-  conv::Operator ConvolutionalOperator,
+  cutlass::conv::Operator ConvolutionalOperator,
  int ConvDim,
  typename ElementA_,
  typename LayoutA_,
@@ -308,7 +309,7 @@ template <
 class ConvReferenceOperation : public Operation {
 public:
  static Provider const kProvider = Provider_;
-  static conv::Operator const kConvolutionalOperator = ConvolutionalOperator;
+  static cutlass::conv::Operator const kConvolutionalOperator = ConvolutionalOperator;
  static int const kConvDim = ConvDim;

  using ElementA = ElementA_;
@@ -491,7 +492,7 @@ void make_conv_fprop(Manifest &manifest) {
  
  manifest.append(new ConvReferenceOperation<
    Provider::kReferenceHost,
-    conv::Operator::kFprop,
+    cutlass::conv::Operator::kFprop,
    kConvDim,
    ElementA_, LayoutA_,
    ElementB_, LayoutB_,
@@ -504,7 +505,7 @@ void make_conv_fprop(Manifest &manifest) {

  manifest.append(new ConvReferenceOperation<
    Provider::kReferenceDevice,
-    conv::Operator::kFprop,
+    cutlass::conv::Operator::kFprop,
    kConvDim,
    ElementA_, LayoutA_,
    ElementB_, LayoutB_,
@@ -534,7 +535,7 @@ void make_conv_backwards(Manifest &manifest) {
  
  manifest.append(new ConvReferenceOperation<
    Provider::kReferenceHost,
-    conv::Operator::kDgrad,
+    cutlass::conv::Operator::kDgrad,
    kConvDim,
    ElementA_, LayoutA_,
    ElementB_, LayoutB_,
@@ -547,7 +548,7 @@ void make_conv_backwards(Manifest &manifest) {

  manifest.append(new ConvReferenceOperation<
    Provider::kReferenceDevice,
-    conv::Operator::kDgrad,
+    cutlass::conv::Operator::kDgrad,
    kConvDim,
    ElementA_, LayoutA_,
    ElementB_, LayoutB_,
@@ -560,7 +561,7 @@ void make_conv_backwards(Manifest &manifest) {

  manifest.append(new ConvReferenceOperation<
    Provider::kReferenceHost,
-    conv::Operator::kWgrad,
+    cutlass::conv::Operator::kWgrad,
    kConvDim,
    ElementA_, LayoutA_,
    ElementB_, LayoutB_,
@@ -573,7 +574,7 @@ void make_conv_backwards(Manifest &manifest) {

  manifest.append(new ConvReferenceOperation<
    Provider::kReferenceDevice,
-    conv::Operator::kWgrad,
+    cutlass::conv::Operator::kWgrad,
    kConvDim,
    ElementA_, LayoutA_,
    ElementB_, LayoutB_,
--- a/tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h
+++ b/tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h
@@ -67,11 +67,12 @@ public:
  /// Problem structure obtained from problem space
  struct GemmProblem {

-    cutlass::library::GemmUniversalMode mode; 
+    cutlass::library::GemmUniversalMode mode;

    int64_t m;
    int64_t n;
    int64_t k;
+    
    int64_t lda;
    int64_t ldb;
    int64_t ldc;
@@ -93,9 +94,16 @@ public:
    // Methods
    //

-    GemmProblem(): 
+    GemmProblem():
      mode(library::GemmUniversalMode::kGemm),
-      m(16), n(16), k(16), lda(0), ldb(0), ldc(0), split_k_slices(1), batch_count(1),
+      m(16), 
+      n(16), 
+      k(16),
+      lda(0), 
+      ldb(0), 
+      ldc(0), 
+      split_k_slices(1), 
+      batch_count(1),
      raster_order(cutlass::library::RasterOrder::kHeuristic){ }

    /// Parses the problem
@@ -117,7 +125,7 @@ public:
      ProblemSpace const &problem_space);
  };

-  /// Workspace used 
+  /// Workspace used
  struct GemmWorkspace {

    DeviceAllocation *A;
@@ -150,7 +158,7 @@ public:
    // Methods
    //

-    GemmWorkspace(): 
+    GemmWorkspace():
      A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr), problem_count(1) { }
  };

@@ -163,7 +171,7 @@ protected:
  /// GEMM problem obtained from problem space
  GemmProblem problem_;

-  /// Device memory allocations 
+  /// Device memory allocations
  GemmWorkspace gemm_workspace_;

  /// CUTLASS parallel reduction operation to follow this* gemm operation
@@ -190,8 +198,8 @@ public:

  /// Extracts the problem dimensions
  virtual Status initialize_configuration(
-    Options const &options, 
-    PerformanceReport &report, 
+    Options const &options,
+    PerformanceReport &report,
    DeviceContext &device_context,
    library::Operation const *operation,
    ProblemSpace const &problem_space,
@@ -199,8 +207,8 @@ public:

  /// Initializes workspace
  virtual Status initialize_workspace(
-    Options const &options, 
-    PerformanceReport &report, 
+    Options const &options,
+    PerformanceReport &report,
    DeviceContext &device_context,
    library::Operation const *operation,
    ProblemSpace const &problem_space,
@@ -208,7 +216,7 @@ public:

  /// Verifies CUTLASS against references
  virtual bool verify_cutlass(
-    Options const &options,  
+    Options const &options,
    PerformanceReport &report,
    DeviceContext &device_context,
    library::Operation const *operation,
@@ -217,8 +225,8 @@ public:

  /// Measures performance results
  virtual bool profile(
-    Options const &options, 
-    PerformanceReport &report, 
+    Options const &options,
+    PerformanceReport &report,
    DeviceContext &device_context,
    library::Operation const *operation,
    ProblemSpace const &problem_space,
@@ -229,13 +237,13 @@ protected:
  /// Initializes the performance result
  void initialize_result_(
    PerformanceResult &result,
-    Options const &options,  
+    Options const &options,
    library::GemmDescription const &operation_desc,
    ProblemSpace const &problem_space);

  /// Verifies CUTLASS against references
  bool verify_with_cublas_(
-    Options const &options,  
+    Options const &options,
    PerformanceReport &report,
    DeviceContext &device_context,
    library::Operation const *operation,
@@ -244,7 +252,7 @@ protected:

  /// Verifies CUTLASS against host and device references
  bool verify_with_reference_(
-    Options const &options,  
+    Options const &options,
    PerformanceReport &report,
    DeviceContext &device_context,
    library::Operation const *operation,
--- a/tools/profiler/src/device_allocation.cu
+++ b/tools/profiler/src/device_allocation.cu
@@ -1493,7 +1493,6 @@ bool DeviceAllocation::block_compare_equal(
      reinterpret_cast<float_e5m2_t const *>(ptr_A),
      reinterpret_cast<float_e5m2_t const *>(ptr_B), 
      capacity);
-
  case library::NumericTypeID::kF16:
    return reference::device::BlockCompareEqual<half_t>(
      reinterpret_cast<half_t const *>(ptr_A), 
@@ -1633,7 +1632,7 @@ bool DeviceAllocation::block_compare_equal(
      capacity);

  default:
-    throw std::runtime_error("Unsupported numeric type");
+    throw std::runtime_error(std::string("Unsupported numeric type: ") + to_string(numeric_type));
  }
 }

@@ -1662,7 +1661,6 @@ bool DeviceAllocation::block_compare_relatively_equal(
      capacity, 
      static_cast<float_e5m2_t>(epsilon), 
      static_cast<float_e5m2_t>(nonzero_floor));
-
  case library::NumericTypeID::kF16:
    return reference::device::BlockCompareRelativelyEqual<half_t>(
      reinterpret_cast<half_t const *>(ptr_A), 
@@ -2089,8 +2087,12 @@ void DeviceAllocation::write_tensor_csv(
    write_tensor_csv_static_type<cutlass::complex<double> >(out, *this);
    break;

+  case library::NumericTypeID::kVoid:
+    // Not dump anything as it is a empty tensor.
+    break;
+
  default:
-    throw std::runtime_error("Unsupported numeric type");
+    throw std::runtime_error(std::string("Unsupported numeric type: ") + to_string(this->type()) ) ;
  }
 }

@@ -2168,7 +2170,6 @@ void DeviceAllocation::fill(double val = 0.0) {
  case library::NumericTypeID::kFE5M2:
    tensor_fill<float_e5m2_t>(*this, static_cast<float_e5m2_t>(val));
    break;
-
  case library::NumericTypeID::kF16:
    tensor_fill<half_t>(*this, static_cast<half_t>(val));
    break;
@@ -2254,7 +2255,7 @@ void DeviceAllocation::fill(double val = 0.0) {
    break;

  default:
-    throw std::runtime_error("Unsupported numeric type");
+    throw std::runtime_error(std::string("Unsupported numeric type: ") + to_string(this->type()));
  }
 }

--- a/tools/profiler/src/gemm_operation_profiler.cu
+++ b/tools/profiler/src/gemm_operation_profiler.cu
@@ -55,7 +55,7 @@ namespace profiler {
 /////////////////////////////////////////////////////////////////////////////////////////////////

 /// Ctor
-GemmOperationProfiler::GemmOperationProfiler(Options const &options): 
+GemmOperationProfiler::GemmOperationProfiler(Options const &options):
  OperationProfiler(
    options,
    library::OperationKind::kGemm,
@@ -73,7 +73,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
      {ArgumentTypeID::kEnumerated, {"split_k_mode", "split-k-mode"}, "Variant of split K mode(serial, parallel)"},
      {ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
      {ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
-      {ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},       
+      {ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},
    },
    { library::Provider::kCUBLAS}
  ) {
@@ -119,7 +119,7 @@ void GemmOperationProfiler::print_examples(std::ostream &out) const {

    << "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
    << " $ cutlass_profiler --operation=Gemm --cta_m=256 --cta_n=128  --cta_k=32 --save-workspace=incorrect\n\n"
-    
+
    << "Test your changes to gemm kernels with a quick functional test and save results in functional-test.csv:\n"
    << " $ cutlass_profiler  --operation=Gemm \\ \n"
    << "   --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
@@ -150,9 +150,9 @@ Status GemmOperationProfiler::GemmProblem::parse(
  library::GemmDescription const &operation_desc,
  ProblemSpace const &problem_space,
  ProblemSpace::Problem const &problem) {
-    
+
  this->mode = library::GemmUniversalMode::kGemm;
-  
+
  if (!arg_as_int(this->m, "m", problem_space, problem)) {
    // default value
    this->m = 1024;
@@ -162,17 +162,17 @@ Status GemmOperationProfiler::GemmProblem::parse(
    // default value
    this->n = 1024;
  }
-  
+
  if (!arg_as_int(this->k, "k", problem_space, problem)) {
    // default value
    this->k = 1024;
  }
-
+  
  if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
    // default value
    this->split_k_mode = library::SplitKMode::kSerial;
  }
-  
+
  this->mode = library::GemmUniversalMode::kGemm;
  if (this->split_k_mode == library::SplitKMode::kParallel) {
    this->mode = library::GemmUniversalMode::kGemmSplitKParallel;
@@ -182,7 +182,7 @@ Status GemmOperationProfiler::GemmProblem::parse(
    // default value
    this->split_k_slices = 1;
  }
-  
+
  if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
    // default value
    this->batch_count = 1;
@@ -194,7 +194,7 @@ Status GemmOperationProfiler::GemmProblem::parse(
    // default value
    this->raster_order = library::RasterOrder::kHeuristic;
  }
-  
+
  if (this->split_k_slices > 1 && this->batch_count > 1) {
    // At least one of these must be one
    return Status::kErrorInvalidProblem;
@@ -217,24 +217,24 @@ Status GemmOperationProfiler::GemmProblem::parse(
  }

  if (!arg_as_scalar(
-    this->alpha, 
-    operation_desc.element_epilogue, 
-    "alpha", 
-    problem_space, 
+    this->alpha,
+    operation_desc.element_epilogue,
+    "alpha",
+    problem_space,
    problem)) {

    if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
      return Status::kErrorInternal;
    }
  }
-  
+
  if (!arg_as_scalar(
-    this->beta, 
-    operation_desc.element_epilogue, 
-    "beta", 
-    problem_space, 
+    this->beta,
+    operation_desc.element_epilogue,
+    "beta",
+    problem_space,
    problem)) {
-    
+
    if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
      return Status::kErrorInternal;
    }
@@ -327,7 +327,7 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
  set_argument(result, "split_k_mode", problem_space, library::to_string(split_k_mode));
  set_argument(result, "split_k_slices", problem_space, split_k_slices);
  set_argument(result, "batch_count", problem_space, batch_count);
-  set_argument(result, "raster_order", problem_space, library::to_string(raster_order)); 
+  set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
  set_argument(result, "alpha", problem_space,
    library::lexical_cast(alpha, operation_desc.element_epilogue));

@@ -339,14 +339,14 @@ void GemmOperationProfiler::GemmProblem::initialize_result(

 /// Extracts the problem dimensions
 Status GemmOperationProfiler::initialize_configuration(
-  Options const &options,  
+  Options const &options,
  PerformanceReport &report,
  DeviceContext &device_context,
  library::Operation const *operation,
  ProblemSpace const &problem_space,
  ProblemSpace::Problem const &problem) {

-  library::GemmDescription const &operation_desc = 
+  library::GemmDescription const &operation_desc =
    static_cast<library::GemmDescription const &>(operation->description());

  if (operation_desc.gemm_kind != library::GemmKind::kUniversal) {
@@ -383,7 +383,6 @@ Status GemmOperationProfiler::initialize_configuration(
  gemm_workspace_.arguments.beta = problem_.beta.data();
  gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
  gemm_workspace_.arguments.raster_order = problem_.raster_order;
-
  // initialize reduction operation for parallel splitKMode
  if (problem_.split_k_mode == library::SplitKMode::kParallel) {
    if (!initialize_reduction_configuration_(operation, problem)) {
@@ -392,14 +391,14 @@ Status GemmOperationProfiler::initialize_configuration(
  }

  initialize_result_(this->model_result_, options, operation_desc, problem_space);
-  
+
  return operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
 }

 /// Initializes the performance result
 void GemmOperationProfiler::initialize_result_(
  PerformanceResult &result,
-  Options const &options,  
+  Options const &options,
  library::GemmDescription const &operation_desc,
  ProblemSpace const &problem_space) {

@@ -451,7 +450,7 @@ bool GemmOperationProfiler::initialize_reduction_configuration_(
  );

  auto reduction_it = library::Singleton::get().operation_table.reduction_operations.find(reduction_key);
- 
+
  if (reduction_it == library::Singleton::get().operation_table.reduction_operations.end()) {
    return false;
  }
@@ -465,7 +464,7 @@ bool GemmOperationProfiler::initialize_reduction_configuration_(

 /// Initializes workspace
 Status GemmOperationProfiler::initialize_workspace(
-  Options const &options,  
+  Options const &options,
  PerformanceReport &report,
  DeviceContext &device_context,
  library::Operation const *operation,
@@ -480,14 +479,14 @@ Status GemmOperationProfiler::initialize_workspace(
    }
  }

-  library::GemmDescription const &operation_desc = 
+  library::GemmDescription const &operation_desc =
    static_cast<library::GemmDescription const &>(operation->description());

  // Compute the number of copies of the problem to avoid L2 camping.
  if (!options.profiling.workspace_count) {
    int64_t bytes = problem_.bytes(operation_desc);
    if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
-      gemm_workspace_.problem_count = 
+      gemm_workspace_.problem_count =
        1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
    }
    else {
@@ -629,7 +628,7 @@ Status GemmOperationProfiler::initialize_workspace(

 /// Verifies CUTLASS against references
 bool GemmOperationProfiler::verify_cutlass(
-  Options const &options,  
+  Options const &options,
  PerformanceReport &report,
  DeviceContext &device_context,
  library::Operation const *operation,
@@ -685,7 +684,7 @@ bool GemmOperationProfiler::verify_cutlass(
  }

  results_.back().status = underlying_operation->run(
-    &gemm_workspace_.arguments, 
+    &gemm_workspace_.arguments,
    gemm_workspace_.host_workspace.data(),
    gemm_workspace_.device_workspace.data());

@@ -748,8 +747,8 @@ bool GemmOperationProfiler::verify_cutlass(
 #endif // #if CUTLASS_ENABLE_CUBLAS

    bool verification_status = verify_with_reference_(options, report, device_context, operation, problem_space, problem);
-    
-    // Update disposition to worst case verification outcome among all 
+
+    // Update disposition to worst case verification outcome among all
    // verification providers which are supported
    bool is_any_verification_run_passed = false;
    for (auto &m : results_.back().verification_map) {
@@ -788,7 +787,7 @@ bool GemmOperationProfiler::verify_cutlass(

 /// Verifies CUTLASS against references
 bool GemmOperationProfiler::verify_with_cublas_(
-  Options const &options,  
+  Options const &options,
  PerformanceReport &report,
  DeviceContext &device_context,
  library::Operation const *operation,
@@ -798,13 +797,13 @@ bool GemmOperationProfiler::verify_with_cublas_(

 #if CUTLASS_ENABLE_CUBLAS

-  library::GemmDescription const &gemm_desc = 
+  library::GemmDescription const &gemm_desc =
    static_cast<library::GemmDescription const &>(operation->description());

  //
  // Construct cuBLAS operators
  //
-    
+
  CublasCreate handle;
  cublasStatus_t status = handle.get_cublas_create_status();

@@ -817,8 +816,8 @@ bool GemmOperationProfiler::verify_with_cublas_(
  std::vector<cublasGemmAlgo_t> algorithms;

  detail::select_cublas_algorithms(
-    algorithms, 
-    options, 
+    algorithms,
+    options,
    gemm_desc);

  if (algorithms.empty()) {
@@ -849,8 +848,8 @@ bool GemmOperationProfiler::verify_with_cublas_(
    gemm_workspace_.arguments.beta = problem_.beta.data();
    gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;

-    detail::cublasGemmExDispatcher gemm_op( 
-      gemm_desc, 
+    detail::cublasGemmExDispatcher gemm_op(
+      gemm_desc,
      gemm_workspace_.configuration,
      gemm_workspace_.arguments,
      algorithms.front()
@@ -884,7 +883,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
    );

    // Save workspace if incorrect
-    if (options.verification.save_workspace == SaveWorkspace::kIncorrect && 
+    if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
      results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {

      save_workspace(
@@ -909,14 +908,14 @@ bool GemmOperationProfiler::verify_with_cublas_(

 /// Verifies CUTLASS against host and device references
 bool GemmOperationProfiler::verify_with_reference_(
-  Options const &options,  
+  Options const &options,
  PerformanceReport &report,
  DeviceContext &device_context,
  library::Operation const *operation,
  ProblemSpace const &problem_space,
  ProblemSpace::Problem const &problem) {

-  library::GemmDescription const &gemm_desc = 
+  library::GemmDescription const &gemm_desc =
    static_cast<library::GemmDescription const &>(operation->description());

  //
@@ -1016,7 +1015,7 @@ bool GemmOperationProfiler::verify_with_reference_(
    results_.back().status = status;

    if (provider == library::Provider::kReferenceHost) {
-      gemm_workspace_.Reference->copy_from_host(ptr_D); 
+      gemm_workspace_.Reference->copy_from_host(ptr_D);
    }

    //
@@ -1031,7 +1030,7 @@ bool GemmOperationProfiler::verify_with_reference_(
    );

    // Save workspace if incorrect
-    if (options.verification.save_workspace == SaveWorkspace::kIncorrect && 
+    if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
      results_.back().verification_map[provider] == Disposition::kIncorrect) {

      save_workspace(
@@ -1050,7 +1049,7 @@ bool GemmOperationProfiler::verify_with_reference_(

 /// Measures performance results
 bool GemmOperationProfiler::profile(
-  Options const &options,  
+  Options const &options,
  PerformanceReport &report,
  DeviceContext &device_context,
  library::Operation const *operation,
@@ -1131,7 +1130,7 @@ Status GemmOperationProfiler::profile_cutlass_(
  Status status;

  for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
-    
+
    int problem_idx = (iteration % gemm_workspace_.problem_count) * problem_.batch_count;

    gemm_workspace_.arguments.A = gemm_workspace_.A->batch_data(problem_idx);
@@ -1184,7 +1183,7 @@ Status GemmOperationProfiler::profile_cutlass_(

  int iteration = 0;
  for (; iteration < Iterations; ++iteration) {
-    
+
    // Iterate over copies of the problem in memory
    int workspace_idx = options.profiling.warmup_iterations + iteration;
    int problem_idx = (workspace_idx % gemm_workspace_.problem_count) * problem_.batch_count;
--- a/tools/util/include/cutlass/util/host_tensor.h
+++ b/tools/util/include/cutlass/util/host_tensor.h
@@ -181,7 +181,7 @@ public:
    device_.reset();
    host_.clear();

-    count = count / kElementsPerStoredVec * kNumStoragePerStoredVec;
+    count = (count + kElementsPerStoredVec - 1) / kElementsPerStoredVec * kNumStoragePerStoredVec;
    host_.resize(count);

    // Allocate memory
--- a/tools/util/include/cutlass/util/packed_stride.hpp
+++ b/tools/util/include/cutlass/util/packed_stride.hpp
@@ -45,6 +45,7 @@ namespace cutlass {
 // Strides without batch mode

 template <class IntT>
+CUTLASS_HOST_DEVICE
 cute::Stride<IntT, cute::Int<1>>
 make_cute_packed_stride(cute::Stride<IntT, cute::Int<1>> s, cute::Shape<int,int,int> shape_MKL) {
  static_assert(std::is_integral_v<IntT>,
@@ -55,6 +56,7 @@ make_cute_packed_stride(cute::Stride<IntT, cute::Int<1>> s, cute::Shape<int,int,
 }

 template <class IntT>
+CUTLASS_HOST_DEVICE
 cute::Stride<cute::Int<1>, IntT>
 make_cute_packed_stride(cute::Stride<cute::Int<1>, IntT> s, cute::Shape<int,int,int> shape_MKL) {
  static_assert(std::is_integral_v<IntT>,
@@ -69,6 +71,7 @@ make_cute_packed_stride(cute::Stride<cute::Int<1>, IntT> s, cute::Shape<int,int,
 // Strides with batch mode

 template <class IntT>
+CUTLASS_HOST_DEVICE
 cute::Stride<IntT, cute::Int<1>, int64_t>
 make_cute_packed_stride(cute::Stride<IntT, cute::Int<1>, int64_t> s, cute::Shape<int,int,int> shape_MKL) {
  static_assert(std::is_integral_v<IntT>,
@@ -86,6 +89,7 @@ make_cute_packed_stride(cute::Stride<IntT, cute::Int<1>, int64_t> s, cute::Shape
 }

 template <class IntT>
+CUTLASS_HOST_DEVICE
 cute::Stride<cute::Int<1>, IntT, int64_t>
 make_cute_packed_stride(cute::Stride<cute::Int<1>, IntT, int64_t> s, cute::Shape<int,int,int> shape_MKL) {
  static_assert(std::is_integral_v<IntT>,
--- a/tools/util/include/cutlass/util/reference/host/gett.hpp
+++ b/tools/util/include/cutlass/util/reference/host/gett.hpp
@@ -257,16 +257,19 @@ void gett_epilogue(
  using BiasBinaryOp = typename EpilogueParams::BiasBinaryOp;

  constexpr bool IsScalingAndAmaxOutputNeeded = 
-      std::is_same_v<ElementD, cutlass::float_e4m3_t> or
-      std::is_same_v<ElementD, cutlass::float_e5m2_t>;
+      cute::is_same_v<ElementD, cutlass::float_e4m3_t> or
+      cute::is_same_v<ElementD, cutlass::float_e5m2_t>;

  constexpr bool IsScalingAndAmaxAuxOutputNeeded =
-      std::is_same_v<ElementAux, cutlass::float_e4m3_t> or
-      std::is_same_v<ElementAux, cutlass::float_e5m2_t>;
+      cute::is_same_v<ElementAux, cutlass::float_e4m3_t> or
+      cute::is_same_v<ElementAux, cutlass::float_e5m2_t>;

  constexpr bool IsReLUAuxNeeded =
-      cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::ReLu<ElementCompute>> and 
+      (cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::ReLu<ElementCompute>> or
+       cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>) and 
      cute::is_same_v<ElementAux, cutlass::uint1b_t>;
+  constexpr bool IsClamp =
+      cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>;

  constexpr bool IsBackpropFusion =
      cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::dGELU<ElementCompute>> or
@@ -276,7 +279,7 @@ void gett_epilogue(
  NumericConverter<ElementCompute, ElementAccumulator> accumulator_converter;
  NumericConverter<ElementCompute, ElementC> source_converter;
  NumericConverter<ElementCompute, ElementBias> bias_converter;
-  NumericConverter<ElementCompute, ElementAux> aux_source_converter;
+  [[maybe_unused]] NumericConverter<ElementCompute, ElementAux> aux_source_converter;

  // Scale related converter
  NumericConverter<ElementCompute, ElementScalar> scale_converter;
@@ -369,7 +372,12 @@ void gett_epilogue(
            }
          }

-          output = activation(output);
+          if constexpr (IsClamp) { // Treat Clamp as ReLU
+            output = activation(output, {0, std::numeric_limits<ElementCompute>::max()});
+          }
+          else {
+            output = activation(output);
+          }
        }

        if constexpr (IsScalingAndAmaxOutputNeeded) {
@@ -436,14 +444,14 @@ void Gemm3x(
  static_assert(cute::rank(typename MainloopParams::LayoutA{}) == cute::rank(typename EpilogueParams::LayoutC{}));

  if constexpr (cute::rank(typename MainloopParams::LayoutA{}) == 2) {
-    Layout layout_A = make_layout_rank3(mainloop_params.A);
-    Layout layout_B = make_layout_rank3(mainloop_params.B);
-    Layout layout_C = make_layout_rank3(epilogue_params.C);
-    Layout layout_D = make_layout_rank3(epilogue_params.D);
-    Layout layout_Aux = make_layout_rank3(epilogue_params.Aux);
-    Layout layout_Bias = make_layout_rank3(epilogue_params.Bias);
-    Layout layout_Valpha = make_layout_rank3(epilogue_params.Valpha);
-    Layout layout_Vbeta = make_layout_rank3(epilogue_params.Vbeta);
+    cute::Layout layout_A = make_layout_rank3(mainloop_params.A);
+    cute::Layout layout_B = make_layout_rank3(mainloop_params.B);
+    cute::Layout layout_C = make_layout_rank3(epilogue_params.C);
+    cute::Layout layout_D = make_layout_rank3(epilogue_params.D);
+    cute::Layout layout_Aux = make_layout_rank3(epilogue_params.Aux);
+    cute::Layout layout_Bias = make_layout_rank3(epilogue_params.Bias);
+    cute::Layout layout_Valpha = make_layout_rank3(epilogue_params.Valpha);
+    cute::Layout layout_Vbeta = make_layout_rank3(epilogue_params.Vbeta);
    
    auto TensorA = make_tensor(mainloop_params.A.data(), layout_A);
    auto TensorB = make_tensor(mainloop_params.B.data(), layout_B);