leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

CUTLASS 3.6.0 (#1850)

Browse Source 
* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-10-09 15:33:27 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 0837a2a00a
commit cc3c29a81a
354 changed files with 105937 additions and 8197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/gemm/device/CMakeLists.txt
+27 -1
View File
@@ -42,6 +42,7 @@ add_custom_target(
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
cutlass_test_unit_gemv_device
cutlass_test_unit_gemm_device_tensorop_sm90
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
)
@@ -255,7 +256,7 @@ cutlass_test_unit_add_executable(
gemm_universal_u8t_f16n_f32t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_s8t_f16n_f16t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_u8t_f16n_f16t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_s8t_bf16n_f32t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_u8t_bf16n_f32t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_s8t_bf16n_bf16t_mixed_input_tensor_op_f32_sm80.cu
@@ -348,6 +349,31 @@ cutlass_test_unit_add_executable(
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm_pingpong.cu
)
# Sparse tests
# Sparse kernels trigger an ICE in gcc 7.5
if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0))
cutlass_test_unit_add_executable(
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
# No batching of source to control compiler memory usage
BATCH_SOURCES ON
BATCH_SIZE 1
sm90_sparse_gemm_s8_s8_s32_tensor_op_s32.cu
sm90_sparse_gemm_f8_f8_f32_tensor_op_f32.cu
sm90_sparse_gemm_f16_f16_f32_tensor_op_f32.cu
sm90_sparse_gemm_tf32_tf32_f32_tensor_op_f32.cu
)
else()
cutlass_test_unit_add_executable(
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
# No batching of source to control compiler memory usage
BATCH_SOURCES ON
BATCH_SIZE 1
)
endif()
# Fused epilogue tests
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_epilogue_fusion_sm90
test/unit/gemm/device/gemm_testbed_3x.hpp
+731 -76
View File
File diff suppressed because it is too large Load Diff
test/unit/gemm/device/gemm_testbed_3x_evt.hpp
+672 -386
View File
File diff suppressed because it is too large Load Diff
test/unit/gemm/device/gemm_testbed_3x_ptr_array.hpp
+26 -25
View File
@@ -76,7 +76,7 @@ enum class ScalarLoc {
ON_DEVICE = 1
};
enum class VectorBeta {
enum class VectorScale {
DISABLED = 0,
ENABLED = 1
};
@@ -556,8 +556,8 @@ struct HostCollectiveDefaultEpilogue {
CheckEquality check_relative_equality = CheckEquality::EXACT;
// Are scalars copied to device memory before kernel launch
ScalarLoc use_device_scalars = ScalarLoc::ON_HOST;
// If per-row scale is enabled and this is true, beta is passed as a host scalar instead of device vector
VectorBeta disable_vector_beta = VectorBeta::DISABLED;
// If per-row scale is enabled and this is disabled, alpha/beta are passed as a host or device scalar instead of device vector
VectorScale vector_scale_mode = VectorScale::DISABLED;
cutlass::Distribution::Kind init_C;
uint64_t seed;
@@ -566,7 +566,7 @@ struct HostCollectiveDefaultEpilogue {
HostCollectiveDefaultEpilogue(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
@@ -850,7 +850,7 @@ struct HostCollectiveEpilogue {
std::vector<cutlass::HostTensor<ElementC, LayoutTagC>> tensors_C;
cutlass::DeviceAllocation<const ElementC *> device_tensors_C;
cutlass::HostTensor<ElementCompute, LayoutTagScalar> norm_constant;
// Outputs
cutlass::HostTensor<ElementAmax, LayoutTagScalar> abs_max_Aux;
cutlass::HostTensor<ElementAmax, LayoutTagScalar> abs_max_D;
@@ -871,8 +871,8 @@ struct HostCollectiveEpilogue {
CheckEquality check_relative_equality = CheckEquality::EXACT;
// Are scalars copied to device memory before kernel launch
ScalarLoc use_device_scalars = ScalarLoc::ON_HOST;
// If per-row scale is enabled and this is true, beta is passed as a host scalar instead of device vector
VectorBeta disable_vector_beta = VectorBeta::DISABLED;
// If per-row scale is enabled and this is disabled, alpha/beta are passed as a host or device scalar instead of device vector
VectorScale vector_scale_mode = VectorScale::DISABLED;
// Random distribution with which to initialize the A/B/C/D/Aux scaling factors
cutlass::Distribution::Kind init_scale = cutlass::Distribution::Uniform;
@@ -885,7 +885,7 @@ struct HostCollectiveEpilogue {
HostCollectiveEpilogue(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
@@ -932,7 +932,7 @@ struct HostCollectiveEpilogue {
if constexpr (IsPerRowScaleEnabled) {
alpha.resize(col_vector_coord);
EXPECT_TRUE(initialize_tensor(alpha.host_view(), init_scale, seed + 2023));
if (disable_vector_beta == VectorBeta::DISABLED) {
if (vector_scale_mode == VectorScale::DISABLED) {
beta.resize(scalar_coord, false);
cutlass::reference::host::TensorFill(beta.host_view(), beta_);
}
@@ -1004,7 +1004,7 @@ struct HostCollectiveEpilogue {
}
stride_Aux = cutlass::make_cute_packed_stride(cutlass::gemm::TagToStrideC_t<LayoutTagAux>{}, cute::make_shape(M, N, 1));
}
static_assert(!IsGroupGemm or (IsGroupGemm and !IsAuxOutEnabled));
if constexpr (IsAuxOutEnabled) {
@@ -1250,9 +1250,10 @@ struct HostCollectiveEpilogue {
else {
fusion_args.alpha = alpha.at(coord_0);
fusion_args.beta = beta.at(coord_0);
fusion_args.alpha_ptr = alpha.device_data();
fusion_args.beta_ptr = beta.device_data(); // if disable_vector_beta is true this is nullptr
fusion_args.alpha_ptr = alpha.device_data();
fusion_args.beta_ptr = beta.device_data();
if constexpr (IsScaleFactorEnabled) {
fusion_args.scale_a = scale_A.at(coord_0);
fusion_args.scale_b = scale_B.at(coord_0);
@@ -1334,9 +1335,9 @@ struct HostCollectiveEpilogue {
return cute::make_tensor(ptr, Aux_layout);
}();
auto Valpha = cute::make_tensor(detail::make_iterator(alpha.host_data()),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cute::make_layout(cute::make_shape(M, N, cute::_1{}), cute::make_stride(cute::_1{}, cute::_0{}, M)));
auto Vbeta = cute::make_tensor(detail::make_iterator(beta.host_data()),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cute::make_layout(cute::make_shape(M, N, cute::_1{}), cute::make_stride(cute::_1{}, cute::_0{}, N)));
cutlass::reference::host::GettEpilogueParams<
ElementScalar,
@@ -1388,7 +1389,7 @@ struct HostCollectiveEpilogue {
if constexpr (IsPerRowScaleEnabled) {
epilogue_params.Valpha = Valpha;
if (disable_vector_beta == VectorBeta::ENABLED) {
if (vector_scale_mode == VectorScale::ENABLED) {
epilogue_params.Vbeta = Vbeta;
}
}
@@ -1442,7 +1443,7 @@ struct TestbedImpl {
TestbedImpl(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
@@ -1450,7 +1451,7 @@ struct TestbedImpl {
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, init_A_, init_B_, seed_)),
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, disable_vector_beta_, init_C_, init_scale_, init_bias_, seed_)) { }
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_C_, init_scale_, init_bias_, seed_)) { }
TestbedImpl(
typename LayoutTagA::Stride stride_factor_A_,
@@ -1459,7 +1460,7 @@ struct TestbedImpl {
typename LayoutTagD::Stride stride_factor_D_,
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
@@ -1467,7 +1468,7 @@ struct TestbedImpl {
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, stride_factor_A_, stride_factor_B_, init_A_, init_B_, seed_)),
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, disable_vector_beta_, init_C_, init_scale_, init_bias_, seed_)) { }
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_C_, init_scale_, init_bias_, seed_)) { }
/// Initializes data structures
bool initialize(ProblemShapeType problem_shapes, ElementScalar alpha_=1.f, ElementScalar beta_=0.f) {
@@ -1501,7 +1502,7 @@ struct TestbedImpl {
file
<< "problem: " << ' ' << M << "x" << N << "x" << K << ", Batch count = " << batch
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
collective_mma_inputs.print_tensors(file, batch);
collective_epilogue.print_tensors(file, batch);
}
@@ -1523,7 +1524,7 @@ struct TestbedImpl {
for (int32_t i = 0; i < L; ++i) {
auto mainloop_params = collective_mma_inputs.to_host_args(problem_shapes, i);
auto epilogue_params = collective_epilogue.to_host_args(problem_shapes, i);
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
passed &= compare_reference(problem_shapes, alpha, beta, i);
@@ -1531,7 +1532,7 @@ struct TestbedImpl {
return passed;
}
/// Determine if the CUDA device is sufficient to run the kernel
/// Determine if the CUDA device is sufficient to run the kernel
bool sufficient() {
//
// Determine SMEM requirements and waive if not satisfied
@@ -1704,14 +1705,14 @@ struct Testbed3x {
Testbed3x(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_DEVICE,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = TestBedImpl::kDefaultSeed)
: impl_(check_relative_equality_, use_device_scalars_, disable_vector_beta_, init_A_, init_B_, init_C_, init_scale_, init_bias_, seed_) {}
: impl_(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_A_, init_B_, init_C_, init_scale_, init_bias_, seed_) {}
/// Executes one test
bool run(
@@ -1734,7 +1735,7 @@ bool TestAll(double alpha = 1.0, double beta = 0.0, CheckEquality check_relative
using ElementScalar = typename Gemm::EpilogueOutputOp::ElementScalar;
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
Testbed3x<Gemm, ActivationFunctor> testbed(check_relative_equality, ScalarLoc::ON_DEVICE, VectorBeta::DISABLED);
Testbed3x<Gemm, ActivationFunctor> testbed(check_relative_equality, ScalarLoc::ON_DEVICE, VectorScale::DISABLED);
int max_alignment = std::max(Gemm::kAlignmentA, Gemm::kAlignmentB);
std::vector<int> problem_size_m = {max_alignment, 512 - 3 * max_alignment};
test/unit/gemm/device/gemm_testbed_3x_tensor_broadcast.hpp
+4 -4
View File
@@ -101,7 +101,7 @@ struct Testbed3xTensorBroadcast {
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
uint64_t seed_ = TestBedImpl::kDefaultSeed
) :
impl_(CheckEquality::EXACT, ScalarLoc::ON_DEVICE, VectorBeta::ENABLED,
impl_(CheckEquality::EXACT, ScalarLoc::ON_DEVICE, VectorScale::ENABLED,
init_A_, init_B_, init_C_, cutlass::Distribution::Uniform, cutlass::Distribution::Uniform, seed_) { }
Testbed3xTensorBroadcast(
@@ -118,7 +118,7 @@ struct Testbed3xTensorBroadcast {
stride_factor_B_,
stride_factor_C_,
stride_factor_D_,
CheckEquality::EXACT, ScalarLoc::ON_HOST, VectorBeta::ENABLED,
CheckEquality::EXACT, ScalarLoc::ON_HOST, VectorScale::ENABLED,
init_A_,
init_B_,
init_C_,
@@ -255,9 +255,9 @@ struct Testbed3xTensorBroadcast {
auto dummy_Aux = cute::make_tensor(static_cast<ElementD*>(nullptr),
cute::make_layout(cute::make_shape(M, N, L), impl_.collective_epilogue.stride_d));
auto dummy_Valpha = cute::make_tensor(static_cast<ElementCompute*>(nullptr),
cute::make_layout(cute::make_shape(M, 1)));
cute::make_layout(cute::make_shape(M, N, 1), cute::make_stride(cute::_1{}, cute::_0{}, M)));
auto dummy_Vbeta = cute::make_tensor(static_cast<ElementCompute*>(nullptr),
cute::make_layout(cute::make_shape(M, 1)));
cute::make_layout(cute::make_shape(M, N, 1), cute::make_stride(cute::_1{}, cute::_0{}, M)));
cutlass::reference::host::GettEpilogueParams<
ElementScalar,
ElementScalar,
test/unit/gemm/device/gemm_universal_bf16t_u8n_bf16t_mixed_input_tensor_op_f32_sm80.cu
+9 -9
View File
@@ -30,7 +30,7 @@
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
@@ -64,14 +64,14 @@ TEST(SM80_Device_GemmUniversal_bf16t_u8n_bf16t_mixed_input_tensor_op_f32, 128x12
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmUniversal<
ElementA,
cutlass::layout::RowMajor,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
ElementA,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
@@ -79,7 +79,7 @@ TEST(SM80_Device_GemmUniversal_bf16t_u8n_bf16t_mixed_input_tensor_op_f32, 128x12
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4, // Stages
8, // AlignmentA
16, // AlignmentB
test/unit/gemm/device/gemm_universal_u8t_bf16n_bf16t_mixed_input_tensor_op_f32_sm80.cu
+9 -9
View File
@@ -30,7 +30,7 @@
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
@@ -64,14 +64,14 @@ TEST(SM80_Device_GemmUniversal_u8t_bf16n_bf16t_mixed_input_tensor_op_f32, 128x12
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmUniversal<
ElementA,
cutlass::layout::RowMajor,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
ElementA,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
@@ -79,7 +79,7 @@ TEST(SM80_Device_GemmUniversal_u8t_bf16n_bf16t_mixed_input_tensor_op_f32, 128x12
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4, // Stages
16, // AlignmentA
8, // AlignmentB
test/unit/gemm/device/sm90_evt_operations.hpp
+151 -97
View File
@@ -37,41 +37,51 @@
//////////////////////////////////////////////////////////////////////////////
/// Host references used for testing
namespace test::gemm::device {
template<class Gemm, class NodeOp, class ...ChildOp>
using HEVT = HostTreeVisitor<Gemm, NodeOp, ChildOp...>;
template<class NodeOp, class ...ChildOp>
using HEVT = HostTreeVisitor<NodeOp, ChildOp...>;
template<class Gemm, class EdgeTuple, class ...Ops>
using HDAG = HostTopoVisitor<Gemm, EdgeTuple, Ops...>;
template<class EdgeTuple, class ...Ops>
using HDAG = HostTopoVisitor<EdgeTuple, Ops...>;
template<class Gemm, class InputTree, class OutputTree, class... AuxOutTrees>
using HST = HostSplitTreeVisitor<Gemm, InputTree, OutputTree, AuxOutTrees...>;
template<class InputTree, class OutputTree, class... AuxOutTrees>
using HST = HostSplitTreeVisitor<InputTree, OutputTree, AuxOutTrees...>;
/// D = alpha * acc + beta * C + AuxLoad
template<class Gemm, class ElementAux, class LayoutAux>
class HostEVTAuxLoad {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using AuxLoadNode = HostAuxLoad<Gemm, false, ElementAux, LayoutAux>;
using TernaryCompute0 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, AuxLoadNode>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, TernaryCompute1>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using AuxLoadNode = HostAuxLoad<ElementAux, LayoutAux, false>;
using TernaryCompute0 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, AuxLoadNode>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, TernaryCompute1>;
};
/// D = alpha * acc + beta * C + per-column bias
template<class Gemm, class ElementBias>
class HostPerColBias {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using RowBroadcastNode = HostRowBroadcast<Gemm, ElementBias>;
using TernaryCompute0 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, RowBroadcastNode>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, TernaryCompute1>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using RowBroadcastNode = HostRowBroadcast<ElementBias>;
using TernaryCompute0 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, RowBroadcastNode>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, TernaryCompute1>;
};
/// D = beta * C + Graph(relu(alpha * acc + aux) + aux)
@@ -79,11 +89,16 @@ public:
template<class Gemm>
class HostEVTDAG {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using AuxLoadNode = HostAuxLoad<Gemm, false, cutlass::half_t, cutlass::layout::RowMajor>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using AuxLoadNode = HostAuxLoad<cutlass::half_t, cutlass::layout::RowMajor, false>;
using DAGNode = HDAG<
Gemm,
float,
cute::tuple<
cute::tuple<>, // 0. alpha
cute::tuple<>, // 1. acc
@@ -95,14 +110,14 @@ public:
ScalarAlpha,
AccFetchNode,
AuxLoadNode,
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostCompute<Gemm, cutlass::epilogue::thread::ReLu>,
HostCompute<Gemm, cutlass::plus>
HostCompute<cutlass::homogeneous_multiply_add>,
HostCompute<cutlass::epilogue::thread::ReLu>,
HostCompute<cutlass::plus>
>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, DAGNode>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, TernaryCompute1>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, DAGNode>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, TernaryCompute1>;
};
/// EVT = alpha * acc + C
@@ -111,19 +126,24 @@ public:
template<class Gemm>
class HostDAGEVT {
public:
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using EVTNode = HEVT<
HostAuxStore<Gemm, false, cutlass::half_t, cutlass::layout::RowMajor>,
HostAuxStore<cutlass::half_t, cutlass::layout::RowMajor, false>,
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 2>,
HostAccumulator<Gemm>,
HostAuxLoad<Gemm, true>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<2>,
HostAccumulator<>,
HostAuxLoad<ElementC, LayoutC, true>
>
>;
using EVTModule = HEVT<
HostAuxStore<Gemm, true>,
HostAuxStore<ElementD, LayoutD, true>,
HDAG<
Gemm,
float,
cute::tuple<
cute::tuple<>, // 0. EVT
cute::tuple<>, // 1. per-row bias
@@ -131,25 +151,30 @@ public:
cute::tuple<cute::_0, cute::_2> // 3. maximum(EVT + per-row bias, EVT)
>,
EVTNode,
HostColBroadcast<Gemm, cutlass::half_t>,
HostCompute<Gemm, cutlass::plus>,
HostCompute<Gemm, cutlass::maximum_with_default_nan_propagation>
HostColBroadcast<cutlass::half_t, cute::Stride<cute::_1,cute::_0,int>>,
HostCompute<cutlass::plus>,
HostCompute<cutlass::maximum_with_default_nan_propagation>
>
>;
};
/// Xreduce(alpha * acc + beta * C)
template<class Gemm, template<class, template <class> class, class> class ReduceOp>
template<class Gemm, class ReduceOp>
class HostReduce {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using BinaryCompute0 = HEVT<HostCompute<Gemm, cutlass::multiplies>, ScalarAlpha, AccFetchNode>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, BinaryCompute0>;
using ReduceNode = HEVT<ReduceOp<Gemm, cutlass::plus, float>, TernaryCompute1>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, ReduceNode>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using BinaryCompute0 = HEVT<HostCompute<cutlass::multiplies>, ScalarAlpha, AccFetchNode>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, BinaryCompute0>;
using ReduceNode = HEVT<ReduceOp, TernaryCompute1>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, ReduceNode>;
};
// Z = scale_a * scale_b * alpha * acc + beta * scale_c * C + per-row bias
@@ -160,25 +185,29 @@ public:
template <class Gemm, template <class> class ActivationFn, class ElementD>
class HostScaledLinCombPerRowBiasEltAct {
public:
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using EVTModule = HEVT<
HostAuxStore<Gemm, true>,
HostAuxStore<ElementD, LayoutD, true>,
HEVT<
HostCompute<Gemm, cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>, // activation(Z) * scaled_d
HostCompute<cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>, // activation(Z) * scaled_d
HEVT<
HostCompute<Gemm, ActivationFn>, // activation(Z)
HostCompute<ActivationFn>, // activation(Z)
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 2>, // scale_c * beta
HostAuxLoad<Gemm, true>, // C
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 2, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_c * beta
HostAuxLoad<ElementC, LayoutC, true>, // C
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 3>, // scale_a * scale_b * alpha
HostAccumulator<Gemm>,
HostColBroadcast<Gemm, ElementD>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 3, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_a * scale_b * alpha
HostAccumulator<>,
HostColBroadcast<ElementD, cute::Stride<cute::_1,cute::_0,int64_t>>
>
>
>,
HostScalarBroadcast<Gemm, 1> // scale_d
HostScalarBroadcast<1> // scale_d
>
>;
};
@@ -197,45 +226,49 @@ public:
template <class Gemm, template <class> class ActivationFn, class ElementD, class ElementAux = ElementD>
class HostScaledLinCombPerRowBiasEltActAmaxAux {
public:
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
template <typename T>
using amax = cutlass::maximum_absolute_value_reduction<T, true>;
using EVTModuleAuxFp8 = HEVT<
HostAuxStore<Gemm, true>,
HST<Gemm,
HostAuxStore<ElementD, LayoutD, true>,
HST<float,
// Z = scale_a * scale_b * alpha * acc + scale_c * beta * C + per-row bias
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 2>, // scale_c * beta
HostAuxLoad<Gemm, true>, // C
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 2, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_c * beta
HostAuxLoad<ElementC, LayoutC, true>, // C
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 3>, // scale_a * scale_b * alpha
HostAccumulator<Gemm>,
HostColBroadcast<Gemm, ElementD>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 3, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_a * scale_b * alpha
HostAccumulator<>,
HostColBroadcast<ElementD, cute::Stride<cute::_1,cute::_0,int64_t>>
>
>,
// D = activation(Z) * scaled_d, amax_d = max(abs(elements in D))
HEVT<
HostCompute<Gemm, cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HostCompute<cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HEVT<
HostScalarReduce<Gemm, amax, float>,
HostScalarReduce<amax, float>,
HEVT<
HostCompute<Gemm, ActivationFn>, //activation(Z) * scaled_d
HostAccumulator<Gemm> // Z
HostCompute<ActivationFn>, //activation(Z) * scaled_d
HostAccumulator<> // Z
>
>,
HostScalarBroadcast<Gemm, 1> // scale_d
HostScalarBroadcast<1> // scale_d
>,
// Aux = Z * scale_aux, amax_aux = max(abs(elements in Aux))
HEVT<
HostAuxStore<Gemm, false, ElementAux, cutlass::layout::RowMajor>,
HostAuxStore<ElementAux, cutlass::layout::RowMajor, false>,
HEVT<
HostCompute<Gemm, cutlass::multiplies>,
HostCompute<cutlass::multiplies>,
HEVT<
HostScalarReduce<Gemm, amax, float>,
HostAccumulator<Gemm>
HostScalarReduce<amax, float>,
HostAccumulator<>
>,
HostScalarBroadcast<Gemm, 1>
HostScalarBroadcast<1>
>
>
>
@@ -243,32 +276,32 @@ public:
using EVTModuleAuxNotFp8 = HEVT<
// D = activation(Z) * scaled_d, amax_d = max(abs(elements in D))
HostAuxStore<Gemm, true>,
HostAuxStore<ElementD, LayoutD, true>,
HEVT<
HostCompute<Gemm, cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HostCompute<cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HEVT<
HostScalarReduce<Gemm, amax, float>,
HostScalarReduce<amax, float>,
HEVT<
HostCompute<Gemm, ActivationFn>, //activation(Z) * scaled_d
HostCompute<ActivationFn>, //activation(Z) * scaled_d
HEVT<
// Aux = Z
HostAuxStore<Gemm, false, ElementAux, cutlass::layout::RowMajor>,
HostAuxStore<ElementAux, cutlass::layout::RowMajor, false>,
// Z = scale_a * scale_b * alpha * acc + scale_c * beta * C + per-row bias
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 2>, // scale_c * beta
HostAuxLoad<Gemm, true>, // C
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 2, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_c * beta
HostAuxLoad<ElementC, LayoutC, true>, // C
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 3>, // scale_a * scale_b * alpha
HostAccumulator<Gemm>,
HostColBroadcast<Gemm, ElementD>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 3, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_a * scale_b * alpha
HostAccumulator<>,
HostColBroadcast<ElementD, cute::Stride<cute::_1,cute::_0,int64_t>>
>
>
>
>
>,
HostScalarBroadcast<Gemm, 1> // scale_d
HostScalarBroadcast<1> // scale_d
>
>;
@@ -314,6 +347,27 @@ using Sm90LinCombAuxLoad =
>
>;
//////////////////////////////////////////////////////////////////////////////
/// D = alpha * acc + beta * C + AuxLoadNoSmem
template<
class EpilogueDescriptor,
class ElementAux,
class StrideAux,
class ElementOutput,
class ElementCompute,
class ElementScalar = ElementCompute,
FloatRoundStyle RoundStyle = FloatRoundStyle::round_to_nearest
>
using Sm90LinCombAuxLoadNoSmem =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + (alpha * acc + bias)
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
Sm90AuxLoad<0, void, ElementAux, StrideAux, void, void> // aux load
>
>;
//////////////////////////////////////////////////////////////////////////////
/// Example DAG
@@ -386,7 +440,7 @@ using Sm90LinCombDAGEVT =
Sm90SrcFetch<ElementOutput>
>
>,
Sm90ColBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>,
Sm90ColBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias, ElementCompute>,
Sm90Compute<plus, ElementCompute, ElementCompute, RoundStyle>,
Sm90Compute<detail::maximum_with_default_nan_propagation, ElementOutput, ElementCompute, RoundStyle>
>;
@@ -409,7 +463,7 @@ using Sm90LinCombPerColumnBias =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
Sm90RowBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>
Sm90RowBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias, ElementCompute>
>
>;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_aux_load.cu
+54
View File
@@ -118,6 +118,60 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_AuxLoadNoSmemF16_RowMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape_MNK = Shape<_256,_128,_64>;
using ClusterShape_MNK = Shape<_2,_2,_1>;
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
using EpilogueDescriptor = cutlass::epilogue::collective::detail::EpilogueDescriptor<
TileShape_MNK, EpilogueTileType, cutlass::half_t, cutlass::half_t, EpilogueSchedule
>;
using FusionCallbacks = cutlass::epilogue::fusion::Sm90LinCombAuxLoadNoSmem<
EpilogueDescriptor, cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t, float, float>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape_MNK, ClusterShape_MNK,
EpilogueTileType,
float, float,
cutlass::half_t, LayoutC, 8,
cutlass::half_t, LayoutC, 8,
EpilogueSchedule,
FusionCallbacks
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
cutlass::half_t, LayoutA, 8,
cutlass::half_t, LayoutB, 8,
float,
TileShape_MNK, ClusterShape_MNK,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostEVTAuxLoad<
Gemm, cutlass::half_t, cutlass::layout::RowMajor
>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>();
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_AuxLoadF16_ColumnMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_aux_store.cu
+64
View File
@@ -329,6 +329,70 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_VoidC_VoidD_AuxStoreNoSmemF16_RowMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape_MNK = Shape<_256,_128,_64>;
using ClusterShape_MNK = Shape<_2,_2,_1>;
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
using EpilogueDescriptor = cutlass::epilogue::collective::detail::EpilogueDescriptor<
TileShape_MNK, EpilogueTileType, cutlass::half_t, cutlass::half_t, EpilogueSchedule
>;
using namespace cutlass::epilogue::fusion;
constexpr auto RoundStyle = cutlass::FloatRoundStyle::round_to_nearest;
constexpr bool has_c = false;
using EVT_D = decltype(test::gemm::device::select_evt_d<cutlass::half_t, float, has_c>());
using AuxStore = Sm90AuxStore<0, void, cutlass::half_t, RoundStyle, cutlass::layout::RowMajor, void, void>;
constexpr auto select_kernel = [](auto has_c, auto has_d) {
using FusionCallbacks =
cute::conditional_t<decltype(has_d){}, EVT_D, Sm90EVT<AuxStore, EVT_D>>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape_MNK, ClusterShape_MNK,
EpilogueTileType,
float, float,
cute::conditional_t<decltype(has_c){}, cutlass::half_t, void>, LayoutC, 8,
cute::conditional_t<decltype(has_d){}, cutlass::half_t, void>, LayoutC, 8,
EpilogueSchedule,
FusionCallbacks
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
cutlass::half_t, LayoutA, 8,
cutlass::half_t, LayoutB, 8,
float,
TileShape_MNK, ClusterShape_MNK,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue>;
return GemmKernel{};
};
using GemmKernel = decltype(select_kernel(cute::C<has_c>{}, cute::C<true>{}));
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
using GemmKernelWithoutD = decltype(select_kernel(cute::C<has_c>{}, cute::C<false>{}));
using GemmWithoutD = cutlass::gemm::device::GemmUniversalAdapter<GemmKernelWithoutD>;
bool passed = test::gemm::device::testEVTAuxStoreWithoutD<Gemm, GemmWithoutD>();
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_VoidC_VoidD_AuxStoreF16_ColumnMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_reduce.cu
+3 -3
View File
@@ -102,7 +102,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -148,7 +148,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -194,7 +194,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_pingpong_reduce.cu
+3 -3
View File
@@ -102,7 +102,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -148,7 +148,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -194,7 +194,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
+126
View File
@@ -119,6 +119,132 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1_ReLu) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule,
cutlass::epilogue::fusion::LinCombEltAct<cutlass::epilogue::thread::ReLu, ElementC, ElementAccumulator>
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
result = TestAll<Gemm>(1.0, 0.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1_silu) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule,
cutlass::epilogue::fusion::LinCombEltAct<cutlass::epilogue::thread::SiLu, ElementC, ElementAccumulator>
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
result = TestAll<Gemm>(1.0, 0.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1_direct_store) {
// A matrix configuration
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm_pingpong.cu
+64 -1
View File
@@ -119,6 +119,69 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm_pingpong, 128x128x64_2x2x1_gelu) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpong; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule,
cutlass::epilogue::fusion::LinCombEltAct<cutlass::epilogue::thread::GELU, ElementC, ElementAccumulator>
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
result = TestAll<Gemm>(1.0, 0.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm_pingpong, 128x128x64_2x2x1_direct_store) {
// A matrix configuration
@@ -181,4 +244,4 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(result);
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array_pingpong.cu
+1 -1
View File
@@ -179,4 +179,4 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(TestAll<Gemm>(1.0, 0.0));
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f16_f16_f32_tensor_op_f32_rs_cluster_warpspecialized_cooperative.cu
+6 -6
View File
@@ -83,12 +83,12 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative, 128x192x64_
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
@@ -159,12 +159,12 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative, 128x192x64_
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
test/unit/gemm/device/sm90_gemm_f32_f32_f32_tensor_op_f32.cu
+3 -3
View File
@@ -186,13 +186,13 @@ TEST(SM90_Device_Gemm_f32t_f32t_f32n_tensor_op_gmma_f32, 128x128x32_1x1x1_cooper
cutlass::detail::TagToStrideA_t<LayoutA>,
float,
cutlass::detail::TagToStrideB_t<LayoutB>,
decltype(cute::make_tiled_mma(cute::SM90_64x64x8_F32TF32TF32_SS_TN{}, Layout<Shape<_2,_1,_1>>{})),
decltype(cute::make_tiled_mma(cute::SM90_64x64x8_F32TF32TF32_SS_TN<>{}, Layout<Shape<_2,_1,_1>>{})),
cute::SM90_TMA_LOAD,
cute::GMMA::Layout_K_SW128_Atom<tfloat32_t>,
GMMA::Layout_K_SW128_Atom<tfloat32_t>,
void,
cute::identity,
cute::SM90_TMA_LOAD,
cute::GMMA::Layout_K_SW128_Atom<tfloat32_t>,
GMMA::Layout_K_SW128_Atom<tfloat32_t>,
void,
cute::identity
>;
test/unit/gemm/device/sm90_gemm_f8_f8_f32_tensor_op_f32_rs_cluster_warpspecialized_cooperative.cu
+6 -6
View File
@@ -83,12 +83,12 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative, 128x128x1
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
@@ -159,12 +159,12 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative, 128x128x1
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
test/unit/gemm/device/sm90_gemm_f8_f8_f8_tensor_op_fp32_evt.cu
+58
View File
@@ -194,4 +194,62 @@ TEST(SM90_Device_Gemm_f8t_f8n_f8t_tensor_op_gmma_f32_persistent_epilogue, 64x128
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
// Z = scale_a * scale_b * alpha * acc + beta * scale_c * C + per-row bias
// if D is fp8
// D = scale_d * filter_negative_zeros(Z)
// else
// D = filter_negative_zeros(Z)
TEST(SM90_Device_Gemm_f8t_f8n_f8t_tensor_op_gmma_f32_persistent_epilogue, 64x128x128_1x1x1_ScaledLinCombPerRowBiasEltFilter) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape_MNK = Shape<_64,_128,_128>;
using ClusterShape_MNK = Shape<_1,_1,_1>;
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
using FusionCallbacks = cutlass::epilogue::fusion::Sm90ScaledLinCombPerRowBiasEltAct<
TileShape_MNK, // CtaTileShapeMNK
cutlass::epilogue::thread::ElementwiseFilter, // ActivationFn
cutlass::float_e4m3_t, // ElementOutput
float, // ElementCompute
cutlass::float_e4m3_t // ElementBias
>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape_MNK, ClusterShape_MNK,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
cutlass::float_e4m3_t, LayoutC, 16,
cutlass::float_e4m3_t, LayoutC, 16,
EpilogueSchedule,
FusionCallbacks
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
cutlass::float_e4m3_t, LayoutA, 16,
cutlass::float_e4m3_t, LayoutB, 16,
float,
TileShape_MNK, ClusterShape_MNK,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostScaledLinCombPerRowBiasEltAct<
Gemm, cutlass::epilogue::thread::ElementwiseFilter, cutlass::float_e4m3_t
>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_stream_k_scheduler.cu
+1 -1
View File
@@ -104,7 +104,7 @@ test_scheduler(
typename Scheduler::Arguments args{};
// Set up the grid for the problem
dim3 grid = Scheduler::get_grid_shape(problem_shape_mnkl, tile_shape, cluster_shape, hw_info, args);
dim3 grid = Scheduler::get_grid_shape(params, problem_shape_mnkl, tile_shape, cluster_shape, hw_info, args);
auto print_info = [&]() {
std::cout << "Failed with problem size "
test/unit/gemm/device/sm90_sparse_gemm_f16_f16_f32_tensor_op_f32.cu
+255
View File
@@ -0,0 +1,255 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f32t_tensorop_f32, 128x128x64_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f32t_tensorop_f32, 128x128x64_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f32t_tensorop_f32, 128x128x64_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpong
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_bf16t_bf16n_f32t_tensorop_f32, 128x128x128_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::bfloat16_t, LayoutA, 16,
cutlass::bfloat16_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f16t_tensorop_f16, 128x128x32_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_32>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
half_t, half_t,
half_t, LayoutC, 4,
half_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
half_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/sm90_sparse_gemm_f8_f8_f32_tensor_op_f32.cu
+216
View File
@@ -0,0 +1,216 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x128_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x128_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x256_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_256>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x64_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/sm90_sparse_gemm_s8_s8_s32_tensor_op_s32.cu
+216
View File
@@ -0,0 +1,216 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x128_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x128_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x256_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_256>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x64_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpong
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/sm90_sparse_gemm_tf32_tf32_f32_tensor_op_f32.cu
+216
View File
@@ -0,0 +1,216 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_tf16t_tf16n_f32t_tensorop_f32, 128x128x64_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_tf16t_tf16n_f32t_tensorop_f32, 128x128x64_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_tf32t_tf32n_f32t_tensorop_f32, 128x128x32_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_32>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_tf32t_tf32n_f32t_tensorop_f32, 128x128x16_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_16>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpong
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/testbed.h
+68 -28
View File
@@ -208,15 +208,17 @@ struct Testbed {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
if (tensor_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
if (reference_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
if (tensor_D.size() > 1) {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0)
<< "tensor_D (size " << tensor_D.size() << ") has nonpositive norm";
}
if (reference_D.size() > 1) {
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0)
<< "reference_D (size " << reference_D.size() << ") has nonpositive norm";
}
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
EXPECT_TRUE(passed);
EXPECT_TRUE(passed) << "reference_D does not equal tensor_D";
if (!passed) {
@@ -369,9 +371,11 @@ struct Testbed {
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
EXPECT_TRUE(status == cutlass::Status::kSuccess)
<< "gemm_op.initialize returned with error " << to_string(status)
<< ", indicating that this test is not supported. Last CUDA error: "
<< cudaGetErrorString(cudaGetLastError());
if (status != cutlass::Status::kSuccess) {
cudaError_t error = cudaGetLastError();
std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
return true;
}
@@ -379,19 +383,27 @@ struct Testbed {
// Run the GEMM
//
status = gemm_op();
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
try {
status = gemm_op();
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "gemm_op() threw a std::exception: " << e.what();
throw;
}
catch (...) {
EXPECT_TRUE(false) << "gemm_op() threw an exception of unknown type";
throw;
}
EXPECT_TRUE(status == cutlass::Status::kSuccess)
<< "gemm_op failed with error " << to_string(status);
//
// Verify
//
bool passed = this->verify(problem_size, alpha, beta);
if (!passed) {
std::cout << "Error with split_k_slices = " << split_k_slices << ", alpha: " << alpha << std::endl;
}
EXPECT_TRUE(passed) << "Error: split_k_slices = " << split_k_slices
<< ", alpha: " << alpha;
return passed;
}
@@ -470,12 +482,26 @@ bool TestAllGemmBasic(
for (auto beta : problem_beta) {
cutlass::gemm::GemmCoord problem_size(m, n, k);
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
try {
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "TestAllGemmBasic: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: " << e.what();
throw;
}
catch (...) {
EXPECT_TRUE(false) << "TestAllGemmBasic: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: (unknown)";
throw;
}
if (!passed) {
return false;
@@ -570,12 +596,26 @@ bool TestGemmPerf(int iterations = 1) {
cutlass::gemm::GemmCoord problem_size(m, n, k);
for (int i = 0; i < iterations; i++){
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
try {
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "TestGemmPerf: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: " << e.what();
throw;
}
catch (...) {
EXPECT_TRUE(false) << "TestGemmPerf: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: (unknown)";
throw;
}
}
if (!passed) {
test/unit/gemm/device/testbed_gemm_with_broadcast.h
+1 -1
View File
@@ -161,7 +161,7 @@ struct TestbedGemmWithBroadcast {
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_max = 1;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
test/unit/gemm/device/testbed_gemm_with_reduction.h
+1 -1
View File
@@ -143,7 +143,7 @@ struct TestbedGemmWithReduction {
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_max = 1;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
test/unit/gemm/device/testbed_universal.h
+1
View File
@@ -201,6 +201,7 @@ struct TestbedUniversal {
if (!passed) {
/*
std::stringstream fname;
fname << "error_Gemm_device_"
test/unit/gemm/threadblock/mma_multistage.cu
-14
View File
@@ -3006,7 +3006,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3041,7 +3040,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3076,7 +3074,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3111,7 +3108,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3146,7 +3142,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3181,7 +3176,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3216,7 +3210,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3251,7 +3244,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3286,7 +3278,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3321,7 +3312,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3356,7 +3346,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3391,7 +3380,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3426,7 +3414,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3461,7 +3448,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_congruous,
tensor_op_64x64x16_32x64x16_8x8x4_3stage) {
test/unit/gemm/warp/gemm_sm80.cu
-209
View File
@@ -1357,215 +1357,6 @@ TEST(SM80_warp_gemm_tensor_op_crosswise_i4, 128x128x256_16x16x256_16x8x64) {
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_64x64x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 64, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_64x32x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 32, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_32x32x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 32, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_32x16x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 16, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_16x16x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<16, 16, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_64x64x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 64, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_64x32x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 32, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_32x32x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 32, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_32x16x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 16, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_16x16x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<16, 16, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_congruous_f64, 16x16x4_16x16x4_8x8x4) {
using Shape = cutlass::gemm::GemmShape<16, 16, 4>;