leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

CUTLASS 3.4.0 (#1286)

Browse Source 
* CUTLASS 3.4.0

* Update CHANGELOG.md

---------

Co-authored-by: Pradeep Ramani <prramani@nvidia.com>
This commit is contained in:
Pradeep Ramani
2023-12-29 15:21:31 -05:00
committed by GitHub
co-authored by Pradeep Ramani
parent b7508e3379
commit 8236f30675
211 changed files with 11409 additions and 2763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/CMakeLists.txt
+7 -2
View File
@@ -136,6 +136,8 @@ function(cutlass_add_cutlass_library)
EXPORT_NAME ${__EXPORT_NAME}
""
)
target_compile_features(${__NAME} INTERFACE cxx_std_17)
set_target_properties(
${__NAME}
@@ -159,6 +161,8 @@ function(cutlass_add_cutlass_library)
EXPORT_NAME ${__EXPORT_NAME}_static
""
)
target_compile_features(${__NAME}_static INTERFACE cxx_std_17)
if (WIN32)
set(STATIC_OUTPUT_NAME ${__OUTPUT_NAME}.static)
@@ -196,8 +200,8 @@ function(cutlass_add_cutlass_library)
# to the main cutlass library so users automatically get the necessary link
# commands to pull in all kernels by default.
target_link_libraries(${DEFAULT_NAME} INTERFACE ${__NAME})
target_link_libraries(${DEFAULT_NAME}_static INTERFACE ${__NAME}_static)
target_link_libraries(${DEFAULT_NAME} PUBLIC ${__NAME})
target_link_libraries(${DEFAULT_NAME}_static PUBLIC ${__NAME}_static)
endif()
@@ -246,6 +250,7 @@ cutlass_add_cutlass_library(
# For backward compatibility with the old name
add_library(cutlass_lib ALIAS cutlass_library)
add_library(cutlass_lib_static ALIAS cutlass_library_static)
################################################################################
tools/library/include/cutlass/library/handle.h
+7 -8
View File
@@ -65,7 +65,7 @@ private:
/// Size of device workspace in bytes
size_t workspace_size_;
/// Indicates whether scalars are host or device pointers
ScalarPointerMode scalar_pointer_mode_;
@@ -89,7 +89,7 @@ public:
//
// Persistent state accessors
//
/// Returns compute capability of the selected device
int compute_capability() const;
@@ -135,7 +135,7 @@ public:
int K, /// GEMM K dimension
NumericTypeID element_compute, /// Data type of internal accumulation
NumericTypeID element_scalar, /// Data type of alpha/beta scalars
void const *alpha, /// Pointer to alpha scalar
@@ -164,7 +164,7 @@ public:
void * ptr_D, /// Pointer to D matrix
int64_t ldd /// Leading dimension of D matrix
);
/// Executes a GEMM computation: D <= alpha * A*B + beta * C.
//
// Supports batched-strided, batched array or split-K serial or split-K parallel.
@@ -176,7 +176,6 @@ public:
int M, /// GEMM M dimension
int N, /// GEMM N dimension
int K, /// GEMM K dimension
NumericTypeID element_compute, /// Data type of internal accumulation
NumericTypeID element_scalar, /// Data type of alpha/beta scalars
@@ -218,7 +217,7 @@ public:
/// Planar complex GEMM
///
/// Note, all data types are the real-valued base types used by the planar-complex GEMM kernel.
///
///
Status gemm_planar_complex(
int M, /// GEMM M dimension
@@ -245,7 +244,7 @@ public:
ComplexTransform transform_B, /// Complex transformation applied to B matrix
void const * ptr_B_real, /// Pointer to real part of B matrix
void const * ptr_B_imag, /// Pointer to imaginary part of B matrix
void const * ptr_B_imag, /// Pointer to imaginary part of B matrix
int64_t ldb_real, /// Leading dimension of real part of B matrix
int64_t ldb_imag, /// Leading dimension of imaginary part of B matrix
@@ -301,7 +300,7 @@ public:
ComplexTransform transform_A, /// Complex transformation applied to A matrix
void const * const * ptr_A_real, /// Pointer to array containing pointers to real part of A matrices
void const * const * ptr_A_imag, /// Pointer to array containing pointers to imaginary part of A matrices
void const * const * ptr_A_imag, /// Pointer to array containing pointers to imaginary part of A matrices
int64_t lda_real, /// Leading dimension of real part of A matrix
int64_t lda_imag, /// Leading dimension of imaginary part of A matrix
tools/library/include/cutlass/library/library.h
+23 -23
View File
@@ -28,17 +28,17 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*!
/*!
\file
\brief CUTLASS Library is an object-oriented approach to managing operations implemented by CUTLASS.
Generally,
description - compile-time constant parameters used to instantiate an operation
configuration - runtime parameters with computationally expensive initialization
configuration - runtime parameters with computationally expensive initialization
arguments - runtime parameters that may be passed to an initialized operation with low
computational overhead
*/
@@ -87,26 +87,26 @@ public:
virtual OperationDescription const & description() const = 0;
virtual Status can_implement(
void const *configuration,
void const *configuration,
void const *arguments) const = 0;
virtual uint64_t get_host_workspace_size(
void const *configuration) const = 0;
virtual uint64_t get_device_workspace_size(
void const *configuration,
void const *arguments = nullptr) const = 0;
virtual Status initialize(
void const *configuration,
void *host_workspace,
void *device_workspace = nullptr,
void const *configuration,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const = 0;
virtual Status run(
void const *arguments,
void *host_workspace,
void *device_workspace = nullptr,
void *host_workspace,
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const = 0;
};
@@ -217,7 +217,7 @@ using GemmBatchedArguments = GemmArguments;
struct GemmArrayConfiguration {
gemm::GemmCoord problem_size;
/// Leading dimension of A matrix
int64_t lda;
@@ -241,7 +241,7 @@ struct GemmArrayArguments {
void * const *D;
void const *alpha;
void const *beta;
ScalarPointerMode pointer_mode;
ScalarPointerMode pointer_mode;
};
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -264,7 +264,7 @@ struct GemmUniversalConfiguration {
};
struct GemmUniversalArguments {
// NOTE: these are replicated for 3.0 interfaces
// NOTE: these are replicated for 3.0 interfaces
gemm::GemmCoord problem_size;
int batch_count;
@@ -645,8 +645,8 @@ struct SymmArguments {
struct Conv2dConfiguration {
conv::SplitKMode split_k_mode;
/// Conv2d problem size
/// Conv2d problem size
// contains strictly conv2d size (N,H,W,C,K,R,S,P,Q,padding,stride,dilation,mode)
// also includes (split_k_slices, groups)
conv::Conv2dProblemSize problem_size;
@@ -669,8 +669,8 @@ struct Conv2dConfiguration {
struct Conv3dConfiguration {
conv::SplitKMode split_k_mode;
/// Conv2d problem size
/// Conv2d problem size
// contains strictly conv2d size (N,D,H,W,C,K,T,R,S,Z,P,Q,padding,stride,dilation,mode)
// also includes (split_k_slices, groups)
conv::Conv3dProblemSize problem_size;
@@ -688,7 +688,7 @@ struct Conv3dConfiguration {
layout::TensorNDHWC layout_output;
//
// Methods
// Methods
//
// Mapping functions (A,B,C -> activation,filter,output)
@@ -734,7 +734,7 @@ struct ConvArguments {
/// pointer to reordered matrix B
void const *reordered_B;
/// pointer to implicit gemm matrix C
void const *C;
@@ -770,7 +770,7 @@ struct ReductionConfiguration {
int64_t partition_stride;
/// leading dimension of 'w'orkspace operand
int64_t ldw;
int64_t ldw;
/// leading dimension of 's'ource operand
int64_t lds;
tools/library/include/cutlass/library/manifest.h
+5 -1
View File
@@ -90,7 +90,11 @@ public:
Status release();
/// Appends an operation and takes ownership
void append(Operation *operation_ptr);
void append(Operation *operation_ptr) {\
// This function is inline s.t. it is present in generated libraries
// without having to compile or link in manifest.cpp
operations_.emplace_back(operation_ptr);
}
/// Returns an iterator to the first operation
OperationVector const &operations() const;
tools/library/src/gemm_operation_3x.hpp
+3 -3
View File
@@ -257,11 +257,11 @@ protected:
case RasterOrder::kAlongM:
operator_args.scheduler.raster_order = Enum_t::AlongM;
break;
default:
default:
operator_args.scheduler.raster_order = Enum_t::Heuristic;
}
}
return status;
}
@@ -271,7 +271,7 @@ public:
Status can_implement(
void const *configuration_ptr, void const *arguments_ptr) const override {
GemmUniversalConfiguration const *configuration =
GemmUniversalConfiguration const *configuration =
static_cast<GemmUniversalConfiguration const *>(configuration_ptr);
GemmUniversalArguments const *arguments =
static_cast<GemmUniversalArguments const *>(arguments_ptr);
tools/library/src/handle.cu
+41 -42
View File
@@ -32,7 +32,7 @@
/*! \file
\brief CUTLASS Library handle.
*/
#include <iostream>
#include <iostream>
#include <stdexcept>
#include <cstdint>
@@ -47,14 +47,14 @@ namespace library {
/// Constructor
Handle::Handle(
cudaStream_t stream,
cudaStream_t stream,
size_t workspace_size
):
provider_(Provider::kCUTLASS),
stream_(stream),
workspace_(nullptr),
workspace_size_(0),
scalar_pointer_mode_(ScalarPointerMode::kHost),
provider_(Provider::kCUTLASS),
stream_(stream),
workspace_(nullptr),
workspace_size_(0),
scalar_pointer_mode_(ScalarPointerMode::kHost),
last_operation_(nullptr) {
int device_idx = -1;
@@ -94,7 +94,7 @@ Handle::Handle(Handle && handle) {
workspace_ = handle.workspace_;
stream_ = handle.stream_;
scalar_pointer_mode_ = handle.scalar_pointer_mode_;
handle.workspace_ = nullptr;
handle.workspace_size_ = 0;
}
@@ -156,14 +156,14 @@ void Handle::set_workspace_size(size_t bytes) {
if (workspace_) {
cudaFree(workspace_);
}
workspace_ = nullptr;
workspace_size_ = bytes;
if (workspace_size_) {
cudaError_t error = cudaMalloc((void **)&workspace_, workspace_size_);
if (error != cudaSuccess) {
throw std::runtime_error("Failed to allocate workspace");
}
@@ -239,7 +239,7 @@ static int gemm_problem_alignment(
};
for (; max_alignment_in_bytes > 0; max_alignment_in_bytes /= 2) {
bool satisfied = true;
// Can pointers satisfy this?
@@ -260,7 +260,7 @@ static int gemm_problem_alignment(
int max_element_alignment = 0;
for (NumericTypeID type_id : elements) {
int element_alignment = max_alignment_in_bytes * 8 / library::sizeof_bits(type_id);
int element_alignment = max_alignment_in_bytes * 8 / library::sizeof_bits(type_id);
max_element_alignment = std::max(max_element_alignment, element_alignment);
}
@@ -286,7 +286,7 @@ static int gemm_problem_alignment(
/// Find the best kernel in descending order of preference.
static Operation const * find_gemm_operation(
GemmOperationFunctionalMap::const_iterator operators_it,
GemmOperationFunctionalMap::const_iterator operators_it,
GemmPreferenceKey const preference_key) {
auto cc_it = operators_it->second.upper_bound(preference_key);
@@ -363,7 +363,7 @@ Status Handle::gemm(
void * ptr_D, /// Pointer to D matrix
int64_t ldd /// Leading dimension of D matrix
) {
//
// Find the operation
//
@@ -390,7 +390,7 @@ Status Handle::gemm(
if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
return cutlass::Status::kErrorNotSupported;
}
if (operators_it->second.empty()) {
return cutlass::Status::kErrorNotSupported;
}
@@ -403,7 +403,7 @@ Status Handle::gemm(
int const kMaximumAlignmentSize = 16;
int alignment = gemm_problem_alignment(
M, N, K,
M, N, K,
element_A, ptr_A, lda, 0,
element_B, ptr_B, ldb, 0,
element_C, ptr_C, ldc, 0,
@@ -491,7 +491,6 @@ Status Handle::gemm_universal(
int M, /// GEMM M dimension
int N, /// GEMM N dimension
int K, /// GEMM K dimension
NumericTypeID element_compute, /// Data type of internal accumulation
NumericTypeID element_scalar, /// Data type of alpha/beta scalars
@@ -529,7 +528,7 @@ Status Handle::gemm_universal(
int64_t batch_stride_C, /// Batch stride of C operand
int64_t batch_stride_D /// Batch stride of D operand
) {
//
// Find the operation
//
@@ -556,7 +555,7 @@ Status Handle::gemm_universal(
if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
return cutlass::Status::kErrorNotSupported;
}
if (operators_it->second.empty()) {
return cutlass::Status::kErrorNotSupported;
}
@@ -576,14 +575,14 @@ Status Handle::gemm_universal(
// Ignore alignment of pointers to pointers. We can't check this from the host,
// as each batch index has its own pointer in device memory.
if (mode == GemmUniversalMode::kArray) {
ptr_A_check = nullptr;
ptr_B_check = nullptr;
ptr_C_check = nullptr;
ptr_D_check = nullptr;
ptr_A_check = nullptr;
ptr_B_check = nullptr;
ptr_C_check = nullptr;
ptr_D_check = nullptr;
}
int alignment = gemm_problem_alignment(
M, N, K,
M, N, K,
element_A, ptr_A_check, lda, 0,
element_B, ptr_B_check, ldb, 0,
element_C, ptr_C_check, ldc, 0,
@@ -758,7 +757,7 @@ Status Handle::gemm_planar_complex(
if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
return cutlass::Status::kErrorNotSupported;
}
if (operators_it->second.empty()) {
return cutlass::Status::kErrorNotSupported;
}
@@ -772,14 +771,14 @@ Status Handle::gemm_planar_complex(
int alignment = std::max(
gemm_problem_alignment(
M, N, K,
M, N, K,
element_A, ptr_A_real, lda_real, batch_stride_A_real,
element_B, ptr_B_real, ldb_real, batch_stride_B_real,
element_C, ptr_C_real, ldc_real, batch_stride_C_real,
ptr_D_real, ldd_real, batch_stride_D_real, kMaximumAlignmentSize
),
gemm_problem_alignment(
M, N, K,
M, N, K,
element_A, ptr_A_imag, lda_imag, batch_stride_A_imag,
element_B, ptr_B_imag, ldb_imag, batch_stride_B_imag,
element_C, ptr_C_imag, ldc_imag, batch_stride_C_imag,
@@ -928,7 +927,7 @@ Status Handle::gemm_planar_complex_array(
int64_t ldd_real, /// Leading dimension of real part of D matrix
int64_t ldd_imag /// Leading dimension of imaginary part of D matrix
) {
//
// Find the operation
//
@@ -955,7 +954,7 @@ Status Handle::gemm_planar_complex_array(
if (operators_it == Singleton::get().operation_table.gemm_operations.end()) {
return cutlass::Status::kErrorNotSupported;
}
if (operators_it->second.empty()) {
return cutlass::Status::kErrorNotSupported;
}
@@ -969,14 +968,14 @@ Status Handle::gemm_planar_complex_array(
int alignment = std::max(
gemm_problem_alignment(
expected_M, expected_N, expected_K,
expected_M, expected_N, expected_K,
element_A, nullptr, lda_real, 0,
element_B, nullptr, ldb_real, 0,
element_C, nullptr, ldc_real, 0,
nullptr, ldd_real, 0, kMaximumAlignmentSize
),
gemm_problem_alignment(
expected_M, expected_N, expected_K,
expected_M, expected_N, expected_K,
element_A, nullptr, lda_imag, 0,
element_B, nullptr, ldb_imag, 0,
element_C, nullptr, ldc_imag, 0,
@@ -1066,7 +1065,7 @@ Status Handle::gemm_planar_complex_array(
/// Finds conv operation instances with Conv::ElementC = Reduction::ElementWorkspace
Operation const* find_conv_operation_for_parallel_reduction(Operation const *operation) {
ConvDescription const &conv_desc =
ConvDescription const &conv_desc =
static_cast<ConvDescription const &>(operation->description());
// if the curren conv operation accumulator and output data type match return operation
@@ -1077,19 +1076,19 @@ Operation const* find_conv_operation_for_parallel_reduction(Operation const *ope
// find conv operation to match conv output and reduction workspace data type
ConvFunctionalKey key(
library::Provider::kCUTLASS,
conv_desc.conv_kind,
conv_desc.conv_kind,
conv_desc.A.element,
conv_desc.A.layout,
conv_desc.B.element,
conv_desc.B.layout,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.C.layout,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.element_epilogue);
// conv operation table for conv2d or conv3d
auto conv_operations = (conv_desc.kind == OperationKind::kConv2d) ?
Singleton::get().operation_table.conv2d_operations :
auto conv_operations = (conv_desc.kind == OperationKind::kConv2d) ?
Singleton::get().operation_table.conv2d_operations :
Singleton::get().operation_table.conv3d_operations;
// find ConvFunctionalKey in convolution operation table
@@ -1098,18 +1097,18 @@ Operation const* find_conv_operation_for_parallel_reduction(Operation const *ope
if (operators_it == conv_operations.end()) {
return nullptr;
}
if (operators_it->second.empty()) {
return nullptr;
}
// conv operation for same compute capability and iterator algorithm
ConvPreferenceKey preference_key(
conv_desc.tile_description.minimum_compute_capability,
conv_desc.tile_description.minimum_compute_capability,
conv_desc.iterator_algorithm);
auto it = operators_it->second.find(preference_key);
if(it == operators_it->second.end()) {
return nullptr;
}
@@ -1129,7 +1128,7 @@ Operation const* find_conv_operation_for_parallel_reduction(Operation const *ope
/// Finds gemm operation instances with Gemm::ElementC = Reduction::ElementWorkspace
Operation const* find_gemm_operation_for_parallel_reduction(Operation const *operation) {
GemmDescription const &gemm_desc =
GemmDescription const &gemm_desc =
static_cast<GemmDescription const &>(operation->description());
// if the curren gemm operation accumulator and output data type match return operation
@@ -1174,7 +1173,7 @@ Operation const* find_gemm_operation_for_parallel_reduction(Operation const *ope
gemm_desc.B.alignment);
GemmPreferenceKey preference_key(
gemm_desc.tile_description.minimum_compute_capability,
gemm_desc.tile_description.minimum_compute_capability,
alignment);
auto it = operators_it->second.find(preference_key);
tools/library/src/manifest.cpp
-5
View File
@@ -77,11 +77,6 @@ Status Manifest::release() {
return Status::kSuccess;
}
/// Appends an operation and takes ownership
void Manifest::append(Operation *operation_ptr) {
operations_.emplace_back(operation_ptr);
}
/// Returns an iterator to the first operation
OperationVector const & Manifest::operations() const {
return operations_;
tools/library/src/reference/conv_reference_operation.h
+12 -11
View File
@@ -45,6 +45,7 @@
#include "cutlass/library/util.h"
#include "library_internal.h"
#include "cutlass/conv/convolution.h"
#include "cutlass/util/reference/host/convolution.h"
#include "cutlass/util/reference/device/convolution.h"
@@ -59,7 +60,7 @@ namespace detail {
template <
Provider kProvider,
conv::Operator ConvolutionalOperator,
cutlass::conv::Operator ConvolutionalOperator,
int ConvDim,
typename ElementA_,
typename LayoutA_,
@@ -77,7 +78,7 @@ struct ConvReferenceDispatcher;
/// Dispatcher for Conv2d (partially specialized for kConvDim == 2)
template <
Provider kProvider,
conv::Operator kConvolutionalOperator,
cutlass::conv::Operator kConvolutionalOperator,
typename ElementA,
typename LayoutA,
typename ElementB,
@@ -193,7 +194,7 @@ struct ConvReferenceDispatcher<
/// Dispatcher for Conv3d (partially specialized for kConvDim == 3)
template <
Provider kProvider,
conv::Operator kConvolutionalOperator,
cutlass::conv::Operator kConvolutionalOperator,
typename ElementA,
typename LayoutA,
typename ElementB,
@@ -292,7 +293,7 @@ struct ConvReferenceDispatcher<
template <
Provider Provider_,
conv::Operator ConvolutionalOperator,
cutlass::conv::Operator ConvolutionalOperator,
int ConvDim,
typename ElementA_,
typename LayoutA_,
@@ -308,7 +309,7 @@ template <
class ConvReferenceOperation : public Operation {
public:
static Provider const kProvider = Provider_;
static conv::Operator const kConvolutionalOperator = ConvolutionalOperator;
static cutlass::conv::Operator const kConvolutionalOperator = ConvolutionalOperator;
static int const kConvDim = ConvDim;
using ElementA = ElementA_;
@@ -491,7 +492,7 @@ void make_conv_fprop(Manifest &manifest) {
manifest.append(new ConvReferenceOperation<
Provider::kReferenceHost,
conv::Operator::kFprop,
cutlass::conv::Operator::kFprop,
kConvDim,
ElementA_, LayoutA_,
ElementB_, LayoutB_,
@@ -504,7 +505,7 @@ void make_conv_fprop(Manifest &manifest) {
manifest.append(new ConvReferenceOperation<
Provider::kReferenceDevice,
conv::Operator::kFprop,
cutlass::conv::Operator::kFprop,
kConvDim,
ElementA_, LayoutA_,
ElementB_, LayoutB_,
@@ -534,7 +535,7 @@ void make_conv_backwards(Manifest &manifest) {
manifest.append(new ConvReferenceOperation<
Provider::kReferenceHost,
conv::Operator::kDgrad,
cutlass::conv::Operator::kDgrad,
kConvDim,
ElementA_, LayoutA_,
ElementB_, LayoutB_,
@@ -547,7 +548,7 @@ void make_conv_backwards(Manifest &manifest) {
manifest.append(new ConvReferenceOperation<
Provider::kReferenceDevice,
conv::Operator::kDgrad,
cutlass::conv::Operator::kDgrad,
kConvDim,
ElementA_, LayoutA_,
ElementB_, LayoutB_,
@@ -560,7 +561,7 @@ void make_conv_backwards(Manifest &manifest) {
manifest.append(new ConvReferenceOperation<
Provider::kReferenceHost,
conv::Operator::kWgrad,
cutlass::conv::Operator::kWgrad,
kConvDim,
ElementA_, LayoutA_,
ElementB_, LayoutB_,
@@ -573,7 +574,7 @@ void make_conv_backwards(Manifest &manifest) {
manifest.append(new ConvReferenceOperation<
Provider::kReferenceDevice,
conv::Operator::kWgrad,
cutlass::conv::Operator::kWgrad,
kConvDim,
ElementA_, LayoutA_,
ElementB_, LayoutB_,