CUTLASS 3.6.0 (#1850)
* v3.6 * update changelog * update readme * fix typo * fixing typos * hopper gemm with weight prefetch --------- Co-authored-by: yuzhai <yuzhai@nvidia.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
co-authored by
yuzhai
Haicheng Wu
parent
0837a2a00a
commit
cc3c29a81a
@@ -251,10 +251,10 @@ template <
|
||||
/// when output layout is interleaved.
|
||||
bool AccumulatorsInRowMajor>
|
||||
struct DefaultMmaTensorOp<
|
||||
WarpShape_,
|
||||
WarpShape_,
|
||||
GemmShape<16, 8, 16>, // InstructionShape
|
||||
ElementA, // Element type of A matrix in Global Memory
|
||||
LayoutA, // Layout of A matrix in Global Memory
|
||||
LayoutA, // Layout of A matrix in Global Memory
|
||||
ElementB, // Element type of B matrix in Global Memory
|
||||
LayoutB, // Layout of B matrix in Global Memory
|
||||
ElementC, // Element type of C matrix in Global Memory
|
||||
@@ -264,7 +264,7 @@ struct DefaultMmaTensorOp<
|
||||
|
||||
|
||||
// Check if the ElementA and ElementB are of different data types
|
||||
static_assert(!platform::is_same<ElementA, ElementB>::value,
|
||||
static_assert(!platform::is_same<ElementA, ElementB>::value,
|
||||
"DefaultMmaTensorOp with arch::OpMultiplyAddMixedInputUpcast ElementA and ElementB cannot be of the same data type");
|
||||
|
||||
// Data type used for internal computation - use the wider of the two data types for mma.sync operands
|
||||
@@ -276,14 +276,14 @@ struct DefaultMmaTensorOp<
|
||||
using ElementBMma = ElementOperand;
|
||||
using MmaElementC = ElementC;
|
||||
|
||||
// Uses
|
||||
// Uses
|
||||
using Policy = cutlass::gemm::warp::MmaTensorOpPolicy<
|
||||
cutlass::arch::Mma<
|
||||
GemmShape<16, 8, 16>,
|
||||
32,
|
||||
ElementAMma, cutlass::layout::RowMajor,
|
||||
GemmShape<16, 8, 16>,
|
||||
32,
|
||||
ElementAMma, cutlass::layout::RowMajor,
|
||||
ElementBMma, cutlass::layout::ColumnMajor,
|
||||
MmaElementC, cutlass::layout::RowMajor,
|
||||
MmaElementC, cutlass::layout::RowMajor,
|
||||
arch::OpMultiplyAdd
|
||||
>,
|
||||
cutlass::MatrixShape<1, 1> >;
|
||||
@@ -294,7 +294,6 @@ struct DefaultMmaTensorOp<
|
||||
Policy, PartitionsK, AccumulatorsInRowMajor>;
|
||||
};
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Partial Specialization - inputs are mixed types - uses wider datatype internally.
|
||||
@@ -320,10 +319,10 @@ template <
|
||||
/// when output layout is interleaved.
|
||||
bool AccumulatorsInRowMajor>
|
||||
struct DefaultMmaTensorOp<
|
||||
WarpShape_,
|
||||
WarpShape_,
|
||||
GemmShape<16, 8, 32>, // InstructionShape
|
||||
ElementA, // Element type of A matrix in Global Memory
|
||||
LayoutA, // Layout of A matrix in Global Memory
|
||||
LayoutA, // Layout of A matrix in Global Memory
|
||||
ElementB, // Element type of B matrix in Global Memory
|
||||
LayoutB, // Layout of B matrix in Global Memory
|
||||
ElementC, // Element type of C matrix in Global Memory
|
||||
@@ -333,11 +332,11 @@ struct DefaultMmaTensorOp<
|
||||
|
||||
|
||||
// Check if the ElementA and ElementB are of different data types
|
||||
static_assert(!platform::is_same<ElementA, ElementB>::value,
|
||||
static_assert(!platform::is_same<ElementA, ElementB>::value,
|
||||
"DefaultMmaTensorOp with arch::OpMultiplyAddMixedInputUpcast ElementA and ElementB cannot be of the same data type");
|
||||
|
||||
// Data type used for internal computation - use the wider of the two data types for mma.sync operands
|
||||
using ElementOperand = typename platform::conditional<(sizeof_bits<ElementA>::value > sizeof_bits<ElementB>::value),
|
||||
using ElementOperand = typename platform::conditional<(sizeof_bits<ElementA>::value > sizeof_bits<ElementB>::value),
|
||||
ElementA, ElementB>::type;
|
||||
|
||||
// Operand datatypes in the internal MMA instruction - use the wider of the two data types
|
||||
@@ -345,14 +344,14 @@ struct DefaultMmaTensorOp<
|
||||
using MmaElementB = ElementOperand;
|
||||
using MmaElementC = ElementC;
|
||||
|
||||
// Uses
|
||||
// Uses
|
||||
using Policy = cutlass::gemm::warp::MmaTensorOpPolicy<
|
||||
cutlass::arch::Mma<
|
||||
GemmShape<16, 8, 32>,
|
||||
32,
|
||||
MmaElementA, cutlass::layout::RowMajor,
|
||||
GemmShape<16, 8, 32>,
|
||||
32,
|
||||
MmaElementA, cutlass::layout::RowMajor,
|
||||
MmaElementB, cutlass::layout::ColumnMajor,
|
||||
MmaElementC, cutlass::layout::RowMajor,
|
||||
MmaElementC, cutlass::layout::RowMajor,
|
||||
arch::OpMultiplyAddSaturate
|
||||
>,
|
||||
cutlass::MatrixShape<1, 1> >;
|
||||
|
||||
@@ -123,10 +123,8 @@ struct FragmentShuffler <ElementMma_, ElementLoad_,
|
||||
NumElementsInWarpFragment,
|
||||
NumElementsInMmaFragment,
|
||||
Operand::kA,
|
||||
typename platform::enable_if<((sizeof_bits<ElementMma_>::value == 16) &&
|
||||
(sizeof_bits<ElementLoad_>::value == 8)) ||
|
||||
((sizeof_bits<ElementMma_>::value == 8) &&
|
||||
(sizeof_bits<ElementLoad_>::value == 4))>::type> {
|
||||
typename platform::enable_if<(sizeof_bits<ElementMma_>::value /
|
||||
sizeof_bits<ElementLoad_>::value == 2)>::type> {
|
||||
public:
|
||||
using ElementMma = ElementMma_;
|
||||
using ElementLoad = ElementLoad_;
|
||||
@@ -209,10 +207,8 @@ struct FragmentShuffler <ElementMma_, ElementLoad_,
|
||||
NumElementsInWarpFragment,
|
||||
NumElementsInMmaFragment,
|
||||
Operand::kB,
|
||||
typename platform::enable_if<((sizeof_bits<ElementMma_>::value == 16) &&
|
||||
(sizeof_bits<ElementLoad_>::value == 8)) ||
|
||||
((sizeof_bits<ElementMma_>::value == 8) &&
|
||||
(sizeof_bits<ElementLoad_>::value == 4))>::type> {
|
||||
typename platform::enable_if<(sizeof_bits<ElementMma_>::value /
|
||||
sizeof_bits<ElementLoad_>::value == 2)>::type> {
|
||||
public:
|
||||
using ElementMma = ElementMma_;
|
||||
using ElementLoad = ElementLoad_;
|
||||
|
||||
@@ -138,7 +138,7 @@ struct ConvertAndPack<half_t, float, N, Round> {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Structure to compute the matrix product targeting CUDA cores and SIMT math instructions.
|
||||
/// Structure to compute the matrix product targeting Tensor Cores.
|
||||
template <
|
||||
/// Size of the Gemm problem - concept: gemm::GemmShape<>
|
||||
typename Shape_,
|
||||
|
||||
Reference in New Issue
Block a user