leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 3.8 Release (#2059)

Browse Source 
* CUTLASS 3.8 Release

* update

* Update README.md

* Revert "Update README.md"

This reverts commit b353e36fe83e0815f99b44e46c0c95494c44726b.

* update

* update

---------

Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
mihir-awatramani
2025-01-25 02:44:06 -05:00
committed by GitHub
co-authored by Haicheng Wu Haicheng Wu
parent 9eb01fa0b0
commit 389e493055
290 changed files with 91222 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/include/cutlass/library/arch_mappings.h
+12
View File
@@ -119,6 +119,18 @@ template <> struct ArchMap<arch::Sm90, arch::OpClassSparseTensorOp> {
static int const kMax = 90;
};
template <typename OperatorClass> struct ArchMap<arch::Sm100, OperatorClass> {
static int const kMin = 100;
static int const kMax = 1024;
};
template <> struct ArchMap<arch::Sm100, arch::OpClassTensorOp> {
static int const kMin = 100;
static int const kMax = 100;
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace library
tools/library/include/cutlass/library/descriptions.h
+95
View File
@@ -300,6 +300,101 @@ struct GemmDescription : public OperationDescription {
transform_B(transform_B) {}
};
/// Description of all GEMM computations
struct BlockScaledGemmDescription : public OperationDescription {
/// Indicates the kind of GEMM performed
GemmKind gemm_kind;
/// Describes the A operand
TensorDescription A;
/// Describes the B operand
TensorDescription B;
/// Describes the source matrix
TensorDescription C;
/// Describes the destination matrix
TensorDescription D;
/// Describes the SFA operand
TensorDescription SFA;
/// Describes the SFB operand
TensorDescription SFB;
/// Describes the SFD operand
TensorDescription SFD;
/// Describes the data type of the scalars passed to the epilogue
NumericTypeID element_epilogue;
/// Describes the structure of parallel reductions
SplitKMode split_k_mode;
/// Transformation on A operand
ComplexTransform transform_A;
/// Transformation on B operand
ComplexTransform transform_B;
/// Describes the input ScaleFactor VectorSize
int SFVecSize;
/// Describes the Output ScaleFactor VectorSize
int EpilogueSFVecSize;
//
// Methods
//
BlockScaledGemmDescription(
GemmKind gemm_kind = GemmKind::kGemm,
TensorDescription const& A = TensorDescription(),
TensorDescription const& B = TensorDescription(),
TensorDescription const& C = TensorDescription(),
TensorDescription const& D = TensorDescription(),
NumericTypeID element_epilogue = NumericTypeID::kInvalid,
SplitKMode split_k_mode = SplitKMode::kNone,
ComplexTransform transform_A = ComplexTransform::kNone,
ComplexTransform transform_B = ComplexTransform::kNone
):
gemm_kind(gemm_kind),
A(A),
B(B),
C(C),
D(D),
element_epilogue(element_epilogue),
split_k_mode(split_k_mode),
transform_A(transform_A),
transform_B(transform_B) {}
BlockScaledGemmDescription(
OperationDescription op_desc,
GemmKind gemm_kind,
TensorDescription const& A,
TensorDescription const& B,
TensorDescription const& C,
TensorDescription const& D,
NumericTypeID element_epilogue,
SplitKMode split_k_mode,
ComplexTransform transform_A,
ComplexTransform transform_B
):
OperationDescription(op_desc),
gemm_kind(gemm_kind),
A(A),
B(B),
C(C),
D(D),
element_epilogue(element_epilogue),
split_k_mode(split_k_mode),
transform_A(transform_A),
transform_B(transform_B) {}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Description for structured sparse GEMMs.
tools/library/include/cutlass/library/handle.h
+9
View File
@@ -178,6 +178,15 @@ public:
int M, /// GEMM M dimension
int N, /// GEMM N dimension
int K, /// GEMM K dimension
int cluster_m, /// cluster shape M dimension
int cluster_n, /// cluster shape N dimension
int cluster_k, /// cluster shape K dimension
int cluster_m_fallback, /// Fallback cluster shape M dimension
int cluster_n_fallback, /// Fallback cluster shape N dimension
int cluster_k_fallback, /// Fallback cluster shape K dimension
NumericTypeID element_compute, /// Data type of internal accumulation
NumericTypeID element_scalar, /// Data type of alpha/beta scalars
tools/library/include/cutlass/library/library.h
+60
View File
@@ -103,6 +103,7 @@ public:
void *device_workspace = nullptr,
cudaStream_t stream = nullptr) const = 0;
// Originally designed for metadata, but should be useful for FP8/6/4 too.
virtual Status initialize_with_profiler_workspace(
void const *configuration,
void *host_workspace,
@@ -269,6 +270,8 @@ struct GemmUniversalConfiguration {
GemmUniversalMode mode{GemmUniversalMode::kGemm};
gemm::GemmCoord problem_size{};
gemm::GemmCoord cluster_shape{};
gemm::GemmCoord cluster_shape_fallback{};
int batch_count{1};
int64_t lda{0};
@@ -282,6 +285,8 @@ struct GemmUniversalConfiguration {
struct GemmUniversalArguments {
// NOTE: these are replicated for 3.0 interfaces
gemm::GemmCoord problem_size{};
gemm::GemmCoord cluster_shape{};
gemm::GemmCoord cluster_shape_fallback{};
int batch_count{1};
void const *A{nullptr};
@@ -307,13 +312,68 @@ struct GemmUniversalArguments {
// Needed for some 3.x kernels
int sm_count{0};
library::RasterOrder raster_order{};
library::RuntimeDatatype runtime_input_datatype_a{};
library::RuntimeDatatype runtime_input_datatype_b{};
int swizzle_size{1};
int split_k_slices{1};
int device_index{0};
bool use_pdl{false};
};
/// Block Scaled GEMM
//
// OperationKind: kBlockScaledGemm
// GemmKind: Universal
struct BlockScaledGemmArguments {
// NOTE: these are replicated for 3.0 interfaces
gemm::GemmCoord problem_size{};
gemm::GemmCoord cluster_shape{};
gemm::GemmCoord cluster_shape_fallback{};
int batch_count{1};
void const *A{nullptr};
void const *B{nullptr};
void const *SFA{nullptr};
void const *SFB{nullptr};
void const *C{nullptr};
void *D{nullptr};
void *SFD{nullptr};
void const *alpha{nullptr};
void const *beta{nullptr};
ScalarPointerMode pointer_mode{};
// NOTE: these are replicated for 3.0 interfaces
int64_t lda{0};
int64_t ldb{0};
int64_t ldc{0};
int64_t ldd{0};
int64_t batch_stride_A{0};
int64_t batch_stride_B{0};
int64_t batch_stride_C{0};
int64_t batch_stride_D{0};
// Needed for ScaleFactor Generation
void const *norm_constant{nullptr};
// Needed for some 3.x kernels
int sm_count{0};
library::RasterOrder raster_order{};
int swizzle_size{1};
int split_k_slices{1};
library::RuntimeDatatype runtime_input_datatype_a{library::RuntimeDatatype::kStatic};
library::RuntimeDatatype runtime_input_datatype_b{library::RuntimeDatatype::kStatic};
bool use_pdl{false};
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Complex valued GEMM in which real and imaginary parts are separated by a stride
tools/library/include/cutlass/library/operation_table.h
+188
View File
@@ -243,6 +243,191 @@ using GemmOperationFunctionalMap = std::unordered_map<
GemmFunctionalKeyHasher
>;
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
// Data Structures for BlockScaled Gemm Functional Maps
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Tuple uniquely identifying Gemm functional behavior
struct BlockScaledGemmFunctionalKey {
Provider provider;
GemmKind gemm_kind;
OperationKind kind;
NumericTypeID element_compute;
NumericTypeID element_scalar;
NumericTypeID element_A;
LayoutTypeID layout_A;
NumericTypeID element_SFA;
NumericTypeID element_B;
LayoutTypeID layout_B;
NumericTypeID element_SFB;
NumericTypeID element_C;
LayoutTypeID layout_C;
NumericTypeID element_D;
LayoutTypeID layout_D;
NumericTypeID element_SFD;
LayoutTypeID layout_SFD;
int SFVecSize;
int EpilogueSFVecSize;
//
// Methods
//
inline
BlockScaledGemmFunctionalKey(
Provider provider,
GemmKind gemm_kind = GemmKind::kGemm,
OperationKind kind = OperationKind::kBlockScaledGemm,
NumericTypeID element_compute = NumericTypeID::kF32,
NumericTypeID element_scalar = NumericTypeID::kF32,
NumericTypeID element_A = NumericTypeID::kF16,
LayoutTypeID layout_A = LayoutTypeID::kColumnMajor,
NumericTypeID element_SFA = NumericTypeID::kF16,
NumericTypeID element_B = NumericTypeID::kF16,
LayoutTypeID layout_B = LayoutTypeID::kColumnMajor,
NumericTypeID element_SFB = NumericTypeID::kF16,
NumericTypeID element_C = NumericTypeID::kF16,
LayoutTypeID layout_C = LayoutTypeID::kColumnMajor,
NumericTypeID element_D = NumericTypeID::kF16,
LayoutTypeID layout_D = LayoutTypeID::kColumnMajor,
NumericTypeID element_SFD = NumericTypeID::kF16,
LayoutTypeID layout_SFD = LayoutTypeID::kRowMajor,
int sf_vec_size = 32
, int epilogue_sf_vec_size = 32
):
provider(provider),
gemm_kind(gemm_kind),
kind(kind),
element_compute(element_compute),
element_scalar(element_scalar),
element_A(element_A),
layout_A(layout_A),
element_SFA(element_SFA),
element_B(element_B),
layout_B(layout_B),
element_SFB(element_SFB),
element_C(element_C),
layout_C(layout_C),
element_D(element_D),
layout_D(layout_D),
element_SFD(element_SFD),
layout_SFD(layout_SFD),
SFVecSize(sf_vec_size)
, EpilogueSFVecSize(epilogue_sf_vec_size)
{ }
inline
bool operator==(BlockScaledGemmFunctionalKey const &rhs) const {
return
(provider == rhs.provider) &&
(gemm_kind == rhs.gemm_kind) &&
(kind == rhs.kind) &&
(element_compute == rhs.element_compute) &&
(element_scalar == rhs.element_scalar) &&
(element_A == rhs.element_A) &&
(layout_A == rhs.layout_A) &&
(element_SFA == rhs.element_SFA) &&
(element_B == rhs.element_B) &&
(layout_B == rhs.layout_B) &&
(element_SFB == rhs.element_SFB) &&
(element_C == rhs.element_C) &&
(layout_C == rhs.layout_C) &&
(element_D == rhs.element_D) &&
(layout_D == rhs.layout_D) &&
(element_SFD == rhs.element_SFD) &&
(layout_SFD == rhs.layout_SFD) &&
(SFVecSize == rhs.SFVecSize)
&& (EpilogueSFVecSize == rhs.EpilogueSFVecSize)
;
}
inline
bool operator!=(BlockScaledGemmFunctionalKey const &rhs) const {
return !(*this == rhs);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
inline
std::ostream & operator<<(std::ostream &out, cutlass::library::BlockScaledGemmFunctionalKey const &k) {
out << "{\n"
<< " provider: " << to_string(k.provider) << "\n"
<< " gemm_kind: " << to_string(k.gemm_kind) << "\n"
<< " kind: " << to_string(k.kind) << "\n"
<< " element_compute: " << to_string(k.element_compute) << "\n"
<< " element_scalar: " << to_string(k.element_scalar) << "\n"
<< " element_A: " << to_string(k.element_A) << "\n"
<< " layout_A: " << to_string(k.layout_A) << "\n"
<< " element_SFA: " << to_string(k.element_SFA) << "\n"
<< " element_B: " << to_string(k.element_B) << "\n"
<< " layout_B: " << to_string(k.layout_B) << "\n"
<< " element_SFB: " << to_string(k.element_SFB) << "\n"
<< " element_C: " << to_string(k.element_C) << "\n"
<< " layout_C: " << to_string(k.layout_C) << "\n"
<< " element_D: " << to_string(k.element_D) << "\n"
<< " layout_D: " << to_string(k.layout_D) << "\n"
<< " element_SFD: " << to_string(k.element_SFD) << "\n"
<< " layout_SFD: " << to_string(k.layout_SFD) << "\n"
<< " SFVecSize: " << k.SFVecSize << "\n"
<< "EpilogueSFVecSize: " << k.EpilogueSFVecSize << "\n"
<< "}";
return out;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Hash function for BlockScaledGemmFunctionalKeyHasher
struct BlockScaledGemmFunctionalKeyHasher {
using IntHash = std::hash<int>;
inline
static size_t rotl(size_t key, int shl) {
return (key << shl) | (key >> (sizeof(key)*8u - static_cast<size_t>(shl)));
}
inline
size_t operator()(BlockScaledGemmFunctionalKey const &key) const {
IntHash hash;
return
rotl(hash(int(key.provider)), 1) ^
rotl(hash(int(key.gemm_kind)), 2) ^
rotl(hash(int(key.kind)), 3) ^
rotl(hash(int(key.element_compute)), 4) ^
rotl(hash(int(key.element_scalar)), 5) ^
rotl(hash(int(key.element_A)), 6) ^
rotl(hash(int(key.layout_A)), 7) ^
rotl(hash(int(key.element_SFA)), 8) ^
rotl(hash(int(key.element_B)), 9) ^
rotl(hash(int(key.layout_B)), 10) ^
rotl(hash(int(key.element_SFB)), 11) ^
rotl(hash(int(key.element_C)), 12) ^
rotl(hash(int(key.layout_C)), 13) ^
rotl(hash(int(key.element_D)), 14) ^
rotl(hash(int(key.layout_D)), 15) ^
rotl(hash(int(key.element_SFD)), 16) ^
rotl(hash(int(key.layout_SFD)), 17) ^
rotl(hash(int(key.SFVecSize)), 18) ^
rotl(hash(int(key.EpilogueSFVecSize)), 19)
;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Maps a GemmFunctionalKey onto a vector of Operation * objects expected to be of kind kGemm
using BlockScaledGemmOperationFunctionalMap = std::unordered_map<
BlockScaledGemmFunctionalKey,
GemmOperationVectorMap,
BlockScaledGemmFunctionalKeyHasher
>;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Data Structures for Conv Functional Maps
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -509,6 +694,9 @@ public:
// provider (kCUTLASS)
GemmOperationFunctionalMap gemm_operations;
// provider (kCUTLASS, kReferenceHost, kReferenceDevice)
BlockScaledGemmOperationFunctionalMap block_scaled_gemm_operations;
/// Map of all operations of type kConv2d
// provider (kCUTLASS, kReferenceHost, kReferenceDevice)
ConvOperationFunctionalMap conv2d_operations;
tools/library/include/cutlass/library/types.h
+28
View File
@@ -43,6 +43,7 @@ enum class LayoutTypeID {
kUnknown,
kColumnMajor,
kRowMajor,
kBlockScalingTensor,
kColumnMajorInterleavedK2,
kRowMajorInterleavedK2,
kColumnMajorInterleavedK4,
@@ -83,6 +84,16 @@ enum class NumericTypeID {
kS64,
kFE4M3,
kFE5M2,
kFE2M3,
kFE3M2,
kFE2M1,
kFUE8M0,
kFUE4M3,
kF8,
kF6,
kF4,
kF16,
kBF16,
kTF32,
@@ -131,6 +142,7 @@ enum class Provider {
/// Enumeration indicating the kind of operation
enum class OperationKind {
kGemm,
kBlockScaledGemm,
kRankK,
kRank2K,
kTrmm,
@@ -165,6 +177,7 @@ enum class OpcodeClassID {
kTensorOp,
kWmmaTensorOp,
kSparseTensorOp,
kBlockScaledOp,
kInvalid
};
@@ -188,6 +201,7 @@ enum class MathOperationID {
/// Enumeration indicating what kind of GEMM operation to perform
enum class GemmKind {
kGemm,
kBlockScaledGemm,
kSparse,
kUniversal,
kPlanarComplex,
@@ -251,6 +265,20 @@ enum class EpilogueKind {
kInvalid
};
enum class RuntimeDatatype {
kStatic,
kE4M3,
kE5M2,
kE3M2,
kE2M3,
kE2M1,
kInvalid
};
enum class RasterOrder {
kAlongN,
kAlongM,
tools/library/include/cutlass/library/util.h
+11
View File
@@ -170,6 +170,15 @@ char const *to_string(ConvKind type, bool pretty = false);
template <>
ConvKind from_string<ConvKind>(std::string const &str);
/// Converts a RuntimeDatatype enumerant to a string
char const *to_string(cutlass::library::RuntimeDatatype type, bool pretty = false);
/// Convers a RuntimeDatatype enumerant from a string
template<>
cutlass::library::RuntimeDatatype from_string<cutlass::library::RuntimeDatatype>(std::string const &str);
/// Converts a RasterOrder enumerant to a string
char const *to_string(RasterOrder type, bool pretty = false);
@@ -202,6 +211,8 @@ bool cast_from_uint64(std::vector<uint8_t> &bytes, NumericTypeID type, uint64_t
/// Casts from a real value represented as a double to the destination type. Returns true if successful.
bool cast_from_double(std::vector<uint8_t> &bytes, NumericTypeID type, double src);
NumericTypeID dynamic_datatype_to_id(RuntimeDatatype type);
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace library