leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 3.6.0 (#1850)

Browse Source 
* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-10-09 15:33:27 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 0837a2a00a
commit cc3c29a81a
354 changed files with 105937 additions and 8197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/self_contained_includes/CMakeLists.txt
+95
View File
@@ -30,12 +30,107 @@
# i.e. they can be included in a source file without needing to include other headers before it.
set(header_files_to_check
# cutlass
# cutlass/gemm/kernel
cutlass/gemm/kernel/default_gemm.h
cutlass/gemm/kernel/default_gemm_complex.h
cutlass/gemm/kernel/gemm_universal_decl.h
# cutlass/gemm/kernel/sm90_gemm_warpspecialized.hpp
# cute
cute/config.hpp
cute/int_tuple.hpp
cute/layout.hpp
cute/layout_composed.hpp
cute/pointer.hpp
cute/pointer_base.hpp
cute/pointer_flagged.hpp
cute/pointer_swizzle.hpp
cute/stride.hpp
cute/swizzle.hpp
cute/swizzle_layout.hpp
cute/tensor.hpp
cute/tensor_impl.hpp
cute/tensor_predicate.hpp
cute/underscore.hpp
# cute/algorithm
cute/algorithm/axpby.hpp
cute/algorithm/clear.hpp
# cute/algorithm/cooperative_copy.hpp
cute/algorithm/cooperative_gemm.hpp
# cute/algorithm/copy.hpp
cute/algorithm/fill.hpp
cute/algorithm/functional.hpp
# cute/algorithm/gemm.hpp
cute/algorithm/prefer.hpp
# cute/algorithm/prefetch.hpp
cute/algorithm/tensor_algorithms.hpp
cute/algorithm/tuple_algorithms.hpp
# cute/container
cute/container/alignment.hpp
cute/container/array.hpp
cute/container/array_aligned.hpp
cute/container/array_subbyte.hpp
cute/container/bit_field.hpp
cute/container/cuda_types.hpp
cute/container/packed_tuple.hpp
cute/container/tuple.hpp
cute/container/type_list.hpp
# cute/numeric
cute/numeric/arithmetic_tuple.hpp
cute/numeric/complex.hpp
cute/numeric/int.hpp
cute/numeric/integer_sequence.hpp
cute/numeric/integral_ratio.hpp
cute/numeric/math.hpp
cute/numeric/numeric_types.hpp
cute/numeric/real.hpp
cute/numeric/integral_constant.hpp
# cute/util
cute/util/debug.hpp
cute/util/print.hpp
cute/util/type_traits.hpp
# cute/arch
cute/arch/cluster_sm90.hpp
cute/arch/copy.hpp
cute/arch/copy_sm50.hpp
cute/arch/copy_sm75.hpp
cute/arch/copy_sm80.hpp
cute/arch/copy_sm90.hpp
cute/arch/copy_sm90_desc.hpp
cute/arch/copy_sm90_tma.hpp
cute/arch/mma_sm61.hpp
cute/arch/mma_sm70.hpp
cute/arch/mma_sm75.hpp
cute/arch/mma_sm80.hpp
cute/arch/mma_sm80_sparse.hpp
cute/arch/mma_sm90.hpp
cute/arch/mma_sm90_desc.hpp
cute/arch/mma_sm90_gmma.hpp
cute/arch/mma.hpp
cute/arch/util.hpp
# cute/atom
# cute/atom/copy_atom.hpp
# cute/atom/copy_traits.hpp
# cute/atom/copy_traits_sm50.hpp
# cute/atom/copy_traits_sm75.hpp
# cute/atom/copy_traits_sm80.hpp
# cute/atom/copy_traits_sm90.hpp
# cute/atom/copy_traits_sm90_im2col.hpp
# cute/atom/copy_traits_sm90_tma.hpp
# cute/atom/copy_traits_sm90_tma_swizzle.hpp
cute/atom/mma_atom.hpp
cute/atom/mma_traits.hpp
cute/atom/mma_traits_sm61.hpp
cute/atom/mma_traits_sm70.hpp
cute/atom/mma_traits_sm75.hpp
cute/atom/mma_traits_sm80.hpp
cute/atom/mma_traits_sm90.hpp
cute/atom/mma_traits_sm90_gmma.hpp
)
# for each header in _header_files:
test/unit/CMakeLists.txt
+1 -1
View File
@@ -84,7 +84,7 @@ function(cutlass_test_unit_add_executable NAME)
target_link_libraries(
${NAME}
PUBLIC
GTest::gtest
GTest::gtest
)
else()
target_link_libraries(
test/unit/conv/cache_testbed_output.h
+1 -1
View File
@@ -609,7 +609,7 @@ inline CachedTestKey CreateCachedGemmTestKey(
ElementCompute alpha,
ElementCompute beta,
cutlass::TensorView<ElementA, LayoutA> A,
cutlass::TensorView<ElementA, LayoutB> B,
cutlass::TensorView<ElementB, LayoutB> B,
cutlass::TensorView<ElementC, LayoutC> C
) {
test/unit/conv/device/conv2d_testbed.h
+13 -1
View File
@@ -718,7 +718,7 @@ bool TestAllConv2d(
}
// CUTLASS DGRAD's *strided* specialization does not support split-k mode
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
@@ -734,6 +734,18 @@ bool TestAllConv2d(
cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0),
cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0));
passed = testbed.run(
cutlass::conv::Conv2dProblemSize(
{1, 56, 56, 8}, // input size (NHWC)
{8, 1, 1, 8}, // filter size (KRSC)
{0, 0, 0, 0}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}) // dilation (dilation_h, dilation_w)
.reset_split_k_slices(2),
cutlass::conv::SplitKMode::kSerial,
cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0),
cutlass::from_real<typename ImplicitGemm::ElementCompute>(2.0));
if (!passed) {
return false;
}
test/unit/conv/device/conv3d_testbed.h
+7 -1
View File
@@ -246,12 +246,18 @@ public:
split_k_mode
);
cutlass::Status status = conv3d_op.can_implement(conv3d_args);
if (status != cutlass::Status::kSuccess) {
std::cerr << "can_implement failed for the given problem_size: \n";
return false;
}
// find workspace requirement for parallel split-k reduction
size_t workspace_size = Conv3d::get_workspace_size(conv3d_args);
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
cutlass::Status status = conv3d_op.initialize(conv3d_args, workspace.get());
status = conv3d_op.initialize(conv3d_args, workspace.get());
if (status != cutlass::Status::kSuccess) {
cudaError_t error = cudaGetLastError();
test/unit/conv/device_3x/conv_problem_sizes.hpp
+84 -61
View File
@@ -308,8 +308,8 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kFprop>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 64},
{256, 2, 5, 64},
{2, 16, 16, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
{1, 1},
@@ -319,8 +319,8 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kFprop>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 15, 64},
{256, 2, 5, 64},
{2, 16, 15, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
{2, 3},
@@ -441,7 +441,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kFprop>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 64},
{96, 3, 4, 5, 64},
{96, 3, 4, 5, 64},
{1, 0, 1},
{0, 2, 0},
{1, 1, 1},
@@ -452,7 +452,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kFprop>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 64},
{96, 3, 4, 5, 64},
{96, 3, 4, 5, 64},
{1, 0, 1},
{0, 2, 0},
{2, 2, 3},
@@ -462,6 +462,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kFprop>() {
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Wgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -570,6 +571,28 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kWgrad>() {
{2},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1024, 128},
{640, 1, 128},
{0},
{0},
{1},
{1},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1040, 128},
{640, 1, 128},
{0},
{0},
{1},
{1},
1
});
return problem_shapes;
}
@@ -659,7 +682,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 15, 16, 32},
{256, 2, 5, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
{2, 3},
@@ -670,7 +693,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 32},
{256, 2, 5, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
{1, 1},
@@ -681,7 +704,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 15, 32},
{256, 2, 5, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
{2, 3},
@@ -690,26 +713,26 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 64, 16, 128}, // nhwc
{640, 1, 1, 128}, // krsc
{0, 0}, // padding lower (pad_h, pad_w)
{0, 0}, // padding upper (pad_h, pad_w)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
1 // group
});
cutlass::conv::Mode::kCrossCorrelation,
{2, 64, 16, 128},
{640, 1, 1, 128},
{0, 0},
{0, 0},
{1, 1},
{1, 1},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 65, 16, 128}, // nhwc
{640, 1, 1, 128}, // krsc
{0, 0}, // padding lower (pad_h, pad_w)
{0, 0}, // padding upper (pad_h, pad_w)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
1 // group
});
cutlass::conv::Mode::kCrossCorrelation,
{2, 65, 16, 128},
{640, 1, 1, 128},
{0, 0},
{0, 0},
{1, 1},
{1, 1},
1
});
return problem_shapes;
}
@@ -755,7 +778,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 32},
{96, 3, 4, 5, 32},
{96, 3, 4, 5, 32},
{1, 0, 1},
{0, 2, 0},
{2, 2, 3},
@@ -766,7 +789,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 32},
{96, 3, 4, 5, 32},
{96, 3, 4, 5, 32},
{1, 0, 1},
{0, 2, 0},
{1, 1, 1},
@@ -775,26 +798,26 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>() {
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 64, 16, 128}, // ndhwc
{640, 1, 1, 1, 128}, // ktrsc
{0, 0, 0}, // padding lower (pad_d, pad_h, pad_w)
{0, 0, 0}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
1 // group
});
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 64, 16, 128},
{640, 1, 1, 1, 128},
{0, 0, 0},
{0, 0, 0},
{1, 1, 1},
{1, 1, 1},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 65, 16, 128}, // ndhwc
{640, 1, 1, 1, 128}, // ktrsc
{0, 0, 0}, // padding lower (pad_d, pad_h, pad_w)
{0, 0, 0}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
1 // group
});
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 65, 16, 128},
{640, 1, 1, 1, 128},
{0, 0, 0},
{0, 0, 0},
{1, 1, 1},
{1, 1, 1},
1
});
return problem_shapes;
}
@@ -848,8 +871,8 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
// Filter-K = 16 for predication
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{1, 8, 16},
{64,1, 16},
{1, 8, 16},
{64, 1, 16},
{0},
{0},
{1},
@@ -870,7 +893,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
// N = 7 and K = 256 for a even larger grid
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{7, 8, 256},
{7, 8, 256},
{64, 1, 256},
{0},
{0},
@@ -881,7 +904,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
// 3 filter, no padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 256},
{2, 8, 256},
{64, 3, 256},
{0},
{0},
@@ -892,7 +915,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
// 3 filter, symmetric padding with k % cta_k !=0
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 256},
{2, 8, 256},
{32, 3, 256},
{1},
{1},
@@ -903,7 +926,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
// 4 filter, asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 256},
{2, 8, 256},
{64, 4, 256},
{0},
{1},
@@ -915,7 +938,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 64},
{256, 3, 64},
{256, 3, 64},
{0},
{1},
{1},
@@ -993,7 +1016,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
// N = 7 and K = 256 for a even larger grid
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{7, 8, 8, 256},
{7, 8, 8, 256},
{64, 1, 1, 256},
{0, 0},
{0, 0},
@@ -1004,7 +1027,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
// 3x3 filter, no padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 8, 256},
{2, 8, 8, 256},
{64, 3, 3, 256},
{0, 0},
{0, 0},
@@ -1015,7 +1038,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
// 3x3 filter, symmetric padding with k % cta_k !=0
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 8, 256},
{2, 8, 8, 256},
{32, 3, 3, 256},
{1, 1},
{1, 1},
@@ -1026,7 +1049,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
// 2x5 filter, asymmetric padding 1,0/1,0
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 8, 256},
{2, 8, 8, 256},
{64, 2, 5, 256},
{1, 1},
{0, 0},
@@ -1038,7 +1061,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 64},
{256, 2, 5, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
{1, 1},
@@ -1116,7 +1139,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad, false>() {
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 64},
{64, 3, 4, 5, 96},
{64, 3, 4, 5, 96},
{1, 0, 1},
{0, 2, 0},
{1, 1, 1},
@@ -1247,7 +1270,7 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad, true>() {
{64, 3, 4, 5, 96},
{1, 0, 1},
{0, 2, 0},
{2, 4, 2},
{2, 1, 2},
{4, 2, 3},
1
});
test/unit/conv/device_3x/dgrad/sm90_conv1d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+16
View File
@@ -87,7 +87,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -130,7 +132,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -216,7 +222,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -264,7 +272,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -307,7 +317,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -350,7 +362,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -393,7 +407,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/dgrad/sm90_conv1d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -87,7 +87,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -130,7 +132,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -216,7 +222,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -263,7 +271,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -306,7 +316,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -349,7 +361,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -392,7 +406,9 @@ TEST(SM90_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/dgrad/sm90_conv2d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+16
View File
@@ -88,7 +88,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -132,7 +134,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -176,7 +180,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -220,7 +226,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -268,7 +276,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -312,7 +322,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -356,7 +368,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -400,7 +414,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/dgrad/sm90_conv2d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/dgrad/sm90_conv3d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -130,7 +132,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -174,7 +178,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -218,7 +224,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -266,7 +274,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -310,7 +320,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -354,7 +366,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -398,7 +412,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/dgrad/sm90_conv3d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv1d_fprop_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+17 -1
View File
@@ -84,7 +84,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -127,7 +129,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -169,8 +173,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -213,7 +219,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -261,7 +269,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -304,7 +314,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -347,7 +359,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -390,7 +404,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv1d_fprop_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+19
View File
@@ -84,7 +84,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -127,7 +129,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -170,7 +174,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -213,7 +219,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -260,7 +268,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -303,7 +314,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -346,7 +360,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -389,7 +406,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv1d_fprop_implicit_gemm_s8_s8_s32_tensorop_s32.cu
+21 -1
View File
@@ -55,6 +55,7 @@ using namespace cute;
// Cluster 1x1x1
//
TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 64x64x64_1x1x1) {
using ElementAct = int8_t;
using ElementFlt = int8_t;
using ElementOut = int32_t;
@@ -84,10 +85,13 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 64x
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
@@ -128,7 +132,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 64x
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -171,7 +178,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 64x
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -214,7 +223,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 64x
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -261,7 +273,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 128
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -304,7 +318,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 128
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -347,7 +363,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 128
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -390,7 +408,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_s8nwc_s8nwc_s32nwc_tensor_op_s32, 128
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv1d_fprop_implicit_gemm_tf32_tf32_f32_tensorop_f32.cu
+17
View File
@@ -84,7 +84,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -127,7 +129,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -170,7 +174,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -213,7 +219,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -260,7 +268,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -303,7 +313,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -346,7 +358,9 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -389,7 +403,10 @@ TEST(SM90_device_conv1d_fprop_implicitgemm_tf32nwc_tf32nwc_f32nwc_tensor_op_f32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+17
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -130,7 +132,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -175,7 +179,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -220,7 +226,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -269,7 +277,10 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -314,7 +325,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -359,7 +372,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -404,7 +419,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -130,7 +132,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -175,7 +179,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -220,7 +226,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -269,7 +277,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -314,7 +324,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -359,7 +371,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -404,7 +418,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_s8_s8_s32_tensorop_s32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_s8nhwc_s8nhwc_s32nhwc_tensor_op_s32,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_tf32_tf32_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -130,7 +132,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -175,7 +179,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -220,7 +226,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -269,7 +277,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -314,7 +324,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -359,7 +371,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -404,7 +418,9 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv3d_fprop_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+17 -1
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -128,8 +130,10 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv3d_fprop_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv3d_fprop_implicit_gemm_s8_s8_s32_tensorop_s32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_s8ndhwc_s8ndhwc_s32ndhwc_tensor_op_s3
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/fprop/sm90_conv3d_fprop_implicit_gemm_tf32_tf32_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv3d_fprop_implicitgemm_tf32ndhwc_tf32ndhwc_f32ndhwc_tensor_o
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/testbed_conv.hpp
+9 -6
View File
@@ -120,12 +120,11 @@ struct DenseConvParams {
// get the default arguments without sparse data
auto get_mainloop_arguments(
ProblemShape const& problem_shape,
[[maybe_unused]] ProblemShape const& problem_shape,
thrust::universal_vector<ElementA>& tensor_A,
thrust::universal_vector<ElementB>& tensor_B
) {
auto args = typename Conv::ConvKernel::MainloopArguments {
problem_shape,
tensor_A.data().get(),
tensor_B.data().get(),
};
@@ -298,11 +297,12 @@ struct ConvTestbed {
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
typename Conv::ConvKernel::TileScheduler::Arguments scheduler_args{};
if constexpr (cute::is_same_v<typename Conv::ConvKernel::TileScheduler::Arguments, cutlass::gemm::StreamKScheduler>) {
if constexpr (cute::is_same_v<typename Conv::ConvKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
scheduler_args = { static_cast<int>(splits), static_cast<int>(max_swizzle), raster_order, decomposition_mode };
}
auto mainloop_args = params.get_mainloop_arguments(problem_shape, tensor_A, tensor_B);
auto mainloop_args = params.get_mainloop_arguments(problem_shape, tensor_A, tensor_B);
auto epilogue_args = typename Conv::ConvKernel::EpilogueArguments {
{},
tensor_C.data().get(),
@@ -312,6 +312,7 @@ struct ConvTestbed {
};
auto args = typename Conv::Arguments {
problem_shape,
mainloop_args, // MainloopArguments
epilogue_args, // EpilogueArguments
hw_info,
@@ -615,8 +616,10 @@ bool TestAllConv(double alpha = 1.0, double beta = 0.0, float epsilon = 0.0f
#endif
for (DecompositionMode decomp_mode : decomposition_modes) {
std::vector problem_splits = {Splits{1}};
if (decomp_mode == DecompositionMode::Heuristic || decomp_mode == DecompositionMode::SplitK) {
problem_splits.push_back(Splits{2});
if constexpr (UsesStreamKScheduler) {
if (decomp_mode == DecompositionMode::Heuristic || decomp_mode == DecompositionMode::SplitK) {
problem_splits.push_back(Splits{2});
}
}
for (auto splits : problem_splits) {
test/unit/conv/device_3x/wgrad/sm90_conv1d_wgrad_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+16
View File
@@ -84,7 +84,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -127,7 +129,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -170,7 +174,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -213,7 +219,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -261,7 +269,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -304,7 +314,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -347,7 +359,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -390,7 +404,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f16, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/wgrad/sm90_conv1d_wgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -84,7 +84,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -127,7 +129,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -170,7 +174,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -213,7 +219,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 6
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -260,7 +268,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -303,7 +313,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -346,7 +358,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -389,7 +403,9 @@ TEST(SM90_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 1
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/wgrad/sm90_conv2d_wgrad_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+17
View File
@@ -85,7 +85,10 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +132,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +178,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +224,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +274,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +320,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +366,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +412,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/wgrad/sm90_conv2d_wgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/wgrad/sm90_conv3d_wgrad_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -218,7 +224,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -266,7 +274,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -310,7 +320,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -354,7 +366,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -398,7 +412,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/conv/device_3x/wgrad/sm90_conv3d_wgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+16
View File
@@ -85,7 +85,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -129,7 +131,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -173,7 +177,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -217,7 +223,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -265,7 +273,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -309,7 +319,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -353,7 +365,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
@@ -397,7 +411,9 @@ TEST(SM90_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue
>;
test/unit/core/fast_numeric_conversion.cu
+3
View File
@@ -103,6 +103,7 @@ void run_test_integer_range_all() {
for (int i = 0; i < kN; ++i) {
source.host_view().at({0, i}) = Source(kIntSourceMin + (i % kIntRange));
}
source.sync_device();
@@ -116,12 +117,14 @@ void run_test_integer_range_all() {
// Verify conversion
bool passed = true;
for (int i = 0; i < kN; ++i) {
if(!(float(destination.host_view().at({0, i})) == float(source.host_view().at({0, i})))) {
passed = false;
break;
}
}
EXPECT_TRUE(passed) << " FastNumericArrayConverter failed";
// Print out results for the failed conversion.
test/unit/core/functional.cu
+12 -14
View File
@@ -497,20 +497,18 @@ __global__ void
test_cutlass_maximum(cutlass::half_t const* in1, cutlass::half_t const* in2, cutlass::half_t* out)
{
{
constexpr bool propagate_NaN = true;
cutlass::maximum<cutlass::half_t, propagate_NaN> op;
if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0
&& blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) {
*out = op(*in1, *in2);
}
constexpr bool propagate_NaN = true;
cutlass::maximum<cutlass::half_t, propagate_NaN> op;
if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0
&& blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) {
*out = op(*in1, *in2);
}
{
constexpr bool propagate_NaN = false;
cutlass::maximum<cutlass::half_t, propagate_NaN> op;
if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0
&& blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) {
*out = op(*in1, *in2);
}
}
constexpr bool propagate_NaN = false;
cutlass::maximum<cutlass::half_t, propagate_NaN> op;
if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0
&& blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) {
*out = op(*in1, *in2);
}
}
@@ -543,7 +541,7 @@ TEST(Functional, maximum_half_host_dont_propagate_NaN) {
EXPECT_EQ(result, y);
}
TEST(Function, maximum_half_device) {
TEST(FUnction, maximum_half_device) {
using Tensor = cutlass::HostTensor<cutlass::half_t, cutlass::layout::RowMajor>;
Tensor in1({1, 1});
test/unit/core/numeric_conversion.cu
+126
View File
@@ -217,6 +217,15 @@ TEST(NumericConversion, f32_to_fe4m3_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f32_to_fe4m3_rn_2_elements) {
int const kN = 2;
using Source = float;
const char source_name[] = "float";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f32_to_fe4m3_rn_array) {
int const kN = 27;
using Source = float;
@@ -235,6 +244,15 @@ TEST(NumericConversion, f32_to_fe5m2_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f32_to_fe5m2_rn_2_elements) {
int const kN = 2;
using Source = float;
const char source_name[] = "float";
using Destination = cutlass::float_e5m2_t;
const char dest_name[] = "float_e5m2_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f32_to_fe5m2_rn_array) {
int const kN = 27;
using Source = float;
@@ -253,6 +271,15 @@ TEST(NumericConversion, f16_to_fe4m3_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f16_to_fe4m3_rn_2_elements) {
int const kN = 2;
using Source = cutlass::half_t;
const char source_name[] = "half_t";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f16_to_fe4m3_rn_array) {
int const kN = 27;
using Source = cutlass::half_t;
@@ -271,6 +298,15 @@ TEST(NumericConversion, f16_to_fe5m2_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f16_to_fe5m2_rn_2_elements) {
int const kN = 27;
using Source = cutlass::half_t;
const char source_name[] = "half_t";
using Destination = cutlass::float_e5m2_t;
const char dest_name[] = "float_e5m2_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f16_to_fe5m2_rn_array) {
int const kN = 27;
using Source = cutlass::half_t;
@@ -289,6 +325,15 @@ TEST(NumericConversion, bf16_to_fe4m3_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, bf16_to_fe4m3_rn_2_elements) {
int const kN = 27;
using Source = cutlass::bfloat16_t;
const char source_name[] = "bfloat16_t";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, bf16_to_fe4m3_rn_array) {
int const kN = 27;
using Source = cutlass::bfloat16_t;
@@ -307,6 +352,15 @@ TEST(NumericConversion, bf16_to_fe5m2_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, bf16_to_fe5m2_rn_2_elements) {
int const kN = 27;
using Source = cutlass::bfloat16_t;
const char source_name[] = "bfloat16_t";
using Destination = cutlass::float_e5m2_t;
const char dest_name[] = "float_e5m2_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, bf16_to_fe5m2_rn_array) {
int const kN = 27;
using Source = cutlass::bfloat16_t;
@@ -327,6 +381,15 @@ TEST(NumericConversion, fe4m3_to_fe5m2_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_fe5m2_2_elements) {
int const kN = 27;
using Source = cutlass::float_e4m3_t;
const char source_name[] = "float_e4m3_t";
using Destination = cutlass::float_e5m2_t;
const char dest_name[] = "float_e5m2_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_fe5m2_array) {
int const kN = 27;
using Source = cutlass::float_e4m3_t;
@@ -345,6 +408,15 @@ TEST(NumericConversion, fe5m2_to_fe4m3_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_fe4m3_2_elements) {
int const kN = 27;
using Source = cutlass::float_e5m2_t;
const char source_name[] = "float_e5m2_t";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_fe4m3_array) {
int const kN = 27;
using Source = cutlass::float_e5m2_t;
@@ -375,6 +447,15 @@ TEST(NumericConversion, f32x8_to_s8x8_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_f32_2_elements) {
int const kN = 2;
using Source = cutlass::float_e4m3_t;
const char source_name[] = "float_e4m3_t";
using Destination = float;
const char dest_name[] = "float";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_f32_array) {
int const kN = 27;
using Source = cutlass::float_e4m3_t;
@@ -384,6 +465,15 @@ TEST(NumericConversion, fe4m3_to_f32_array) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_f32_2_elements) {
int const kN = 2;
using Source = cutlass::float_e5m2_t;
const char source_name[] = "float_e5m2_t";
using Destination = float;
const char dest_name[] = "float";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_f32_array) {
int const kN = 27;
using Source = cutlass::float_e5m2_t;
@@ -402,6 +492,15 @@ TEST(NumericConversion, fe4m3_to_f16_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_f16_2_elements) {
int const kN = 2;
using Source = cutlass::float_e4m3_t;
const char source_name[] = "float_e4m3_t";
using Destination = cutlass::half_t;
const char dest_name[] = "half_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_f16_array) {
int const kN = 27;
using Source = cutlass::float_e4m3_t;
@@ -420,6 +519,15 @@ TEST(NumericConversion, fe5m2_to_f16_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_f16_2_elements) {
int const kN = 2;
using Source = cutlass::float_e5m2_t;
const char source_name[] = "float_e5m2_t";
using Destination = cutlass::half_t;
const char dest_name[] = "half_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_f16_array) {
int const kN = 27;
using Source = cutlass::float_e5m2_t;
@@ -438,6 +546,15 @@ TEST(NumericConversion, fe4m3_to_bf16_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_bf16_2_elements) {
int const kN = 2;
using Source = cutlass::float_e4m3_t;
const char source_name[] = "float_e4m3_t";
using Destination = cutlass::bfloat16_t;
const char dest_name[] = "bfloat16_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe4m3_to_bf16_array) {
int const kN = 27;
using Source = cutlass::float_e4m3_t;
@@ -456,6 +573,15 @@ TEST(NumericConversion, fe5m2_to_bf16_rn) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_bf16_2_elements) {
int const kN = 2;
using Source = cutlass::float_e5m2_t;
const char source_name[] = "float_e5m2_t";
using Destination = cutlass::bfloat16_t;
const char dest_name[] = "bfloat16_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_to_bf16_array) {
int const kN = 27;
using Source = cutlass::float_e5m2_t;
test/unit/cute/ampere/cooperative_copy.cu
+2
View File
@@ -43,6 +43,8 @@
#include <thrust/device_vector.h>
#include <cute/tensor.hpp>
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::compose(cute::Swizzle)
#include <cute/numeric/numeric_types.hpp>
using namespace cute;
test/unit/cute/ampere/cooperative_gemm.cu
+2
View File
@@ -32,6 +32,8 @@
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::compose(cute::Swizzle)
#include "../cooperative_gemm_common.hpp"
test/unit/cute/ampere/tiled_cp_async.cu
+1
View File
@@ -42,6 +42,7 @@
#include <thrust/device_vector.h>
#include <cute/tensor.hpp>
#include <cute/swizzle.hpp> // cute::Swizzle
#include "tiled_cp_async_testbed.hpp"
test/unit/cute/core/CMakeLists.txt
+2 -1
View File
@@ -39,6 +39,7 @@ cutlass_test_unit_add_executable(
constants.cpp
core_unit.cpp
domain_distribute.cpp
int_tuple.cpp
inverse_left.cpp
inverse_right.cpp
logical_divide.cpp
@@ -49,8 +50,8 @@ cutlass_test_unit_add_executable(
packed_tuple.cpp
pointer.cpp
reverse.cpp
swizzle_layout.cpp
transform.cpp
tuple.cpp
tuple_find.cpp
int_tuple.cpp
)
test/unit/cute/core/composition.cpp
+6 -3
View File
@@ -29,13 +29,16 @@
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <cute/layout.hpp>
#include <cute/layout_composed.hpp> // cute::composition
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::composition
#include <cute/tensor.hpp>
#include <iostream>
#include <cute/tensor.hpp>
#include "cutlass_unit_test.h"
using namespace cute;
test/unit/cute/core/domain_distribute.cpp
+3 -4
View File
@@ -29,7 +29,7 @@
*
**************************************************************************************************/
#define CUTLASS_DEBUG_TRACE_LEVEL 1
//#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
@@ -41,7 +41,6 @@
using namespace cute;
template <class LayoutA, class LayoutB>
void
test_distribute(LayoutA const& layoutA,
@@ -54,8 +53,8 @@ test_distribute(LayoutA const& layoutA,
CUTLASS_TRACE_HOST(" => ");
CUTLASS_TRACE_HOST(layoutR);
// Test that layout B is softly compatible with layout R
EXPECT_TRUE(softly_compatible(layoutB, layoutR));
EXPECT_TRUE(evenly_divides(layoutB, size(layoutR)));
EXPECT_TRUE(evenly_divides(layoutA, layoutR));
// Post-condition on the codomain of the distribute
for (int i = 0; i < size(layoutR); ++i) {
test/unit/cute/core/int_tuple.cpp
+95 -75
View File
@@ -33,10 +33,10 @@
#include <cute/layout.hpp>
using namespace cute;
TEST(CuTe_core, WeaklyCongruent)
{
using namespace cute;
auto a = _1{};
auto b = _2{};
EXPECT_TRUE (weakly_congruent(a, a));
@@ -83,96 +83,116 @@ TEST(CuTe_core, WeaklyCongruent)
EXPECT_TRUE (weakly_congruent(a2, b3));
}
TEST(CuTe_core, WeaklyCompatible)
template <class A, class B>
auto test_evenly_divides(A const& a, B const& b)
{
using namespace cute;
auto result = evenly_divides(a, b);
// If A and B are static, then result should be as well
if constexpr (is_static<A>::value && is_static<B>::value) {
static_assert(is_static<decltype(result)>::value);
}
// If result is true_type, then confirm divisibillity
if constexpr (is_constant<true, decltype(result)>::value) {
CUTE_STATIC_ASSERT_V(size(a) == size(logical_divide(make_layout(shape(a)), b)));
}
return result;
}
TEST(CuTe_core, Divides)
{
{
auto a = _16{};
auto b = _12{};
auto c = _8{};
EXPECT_TRUE (weakly_compatible(a, a));
EXPECT_TRUE (weakly_compatible(b, b));
EXPECT_TRUE (weakly_compatible(c, c));
EXPECT_FALSE(weakly_compatible(a, b));
EXPECT_FALSE(weakly_compatible(a, c));
EXPECT_TRUE (weakly_compatible(c, a));
EXPECT_TRUE (test_evenly_divides(a, a));
EXPECT_TRUE (test_evenly_divides(b, b));
EXPECT_TRUE (test_evenly_divides(c, c));
EXPECT_FALSE(test_evenly_divides(a, b));
EXPECT_TRUE (test_evenly_divides(a, c));
EXPECT_FALSE(test_evenly_divides(c, a));
auto a0 = Shape<_16>{};
EXPECT_TRUE (weakly_compatible(a0, a0));
EXPECT_TRUE (weakly_compatible(a , a0));
EXPECT_FALSE(weakly_compatible(a0, a ));
EXPECT_TRUE (weakly_compatible(c , a0));
EXPECT_FALSE(weakly_compatible(a0, c ));
EXPECT_FALSE(weakly_compatible(b , a0));
EXPECT_FALSE(weakly_compatible(a0, b ));
EXPECT_TRUE (test_evenly_divides(a0, a0));
EXPECT_TRUE (test_evenly_divides(a , a0));
EXPECT_TRUE (test_evenly_divides(a0, a ));
EXPECT_FALSE(test_evenly_divides(c , a0));
EXPECT_TRUE (test_evenly_divides(a0, c ));
EXPECT_FALSE(test_evenly_divides(b , a0));
EXPECT_FALSE(test_evenly_divides(a0, b ));
auto a1 = Shape<_2,_8>{};
EXPECT_TRUE (weakly_compatible(a1, a1));
EXPECT_TRUE (weakly_compatible(a , a1));
EXPECT_FALSE(weakly_compatible(a0, a1));
EXPECT_FALSE(weakly_compatible(a1, a0));
EXPECT_TRUE (weakly_compatible(a1, Shape<_2,Shape<_2,_4>>{}));
EXPECT_TRUE (test_evenly_divides(a1, a1));
EXPECT_FALSE(test_evenly_divides(a , a1));
EXPECT_FALSE(test_evenly_divides(a0, a1));
EXPECT_FALSE(test_evenly_divides(a1, a0));
EXPECT_FALSE(test_evenly_divides(a1, Shape<_2,Shape<_2,_4>>{}));
auto a2 = Shape<Shape<_2,_8>>{};
EXPECT_TRUE (weakly_compatible(a2, a2));
EXPECT_TRUE (weakly_compatible(a , a2));
EXPECT_TRUE (weakly_compatible(c , a2));
EXPECT_TRUE (weakly_compatible(a0, a2));
EXPECT_FALSE(weakly_compatible(a2, a0));
EXPECT_TRUE (test_evenly_divides(a2, a2));
EXPECT_FALSE(test_evenly_divides(a , a2));
EXPECT_FALSE(test_evenly_divides(c , a2));
EXPECT_FALSE(test_evenly_divides(a0, a2));
EXPECT_TRUE (test_evenly_divides(a2, a0));
auto a3 = Shape<Shape<_2,Shape<_4,_2>>>{};
EXPECT_TRUE (weakly_compatible(a3, a3));
EXPECT_TRUE (weakly_compatible(a , a3));
EXPECT_TRUE (weakly_compatible(c , a3));
EXPECT_TRUE (weakly_compatible(a0, a3));
EXPECT_FALSE(weakly_compatible(a3, a0));
EXPECT_TRUE (weakly_compatible(a2, a3));
EXPECT_FALSE(weakly_compatible(a3, a2));
}
EXPECT_TRUE (test_evenly_divides(a3, a3));
EXPECT_FALSE(test_evenly_divides(a , a3));
EXPECT_FALSE(test_evenly_divides(c , a3));
EXPECT_FALSE(test_evenly_divides(a0, a3));
EXPECT_TRUE (test_evenly_divides(a3, a0));
EXPECT_FALSE(test_evenly_divides(a2, a3));
EXPECT_TRUE (test_evenly_divides(a3, a2));
}
TEST(CuTe_core, SoftlyCompatible)
{
using namespace cute;
{
auto a = 16;
auto b = 12;
auto c = 8;
EXPECT_TRUE (test_evenly_divides(a, a));
EXPECT_TRUE (test_evenly_divides(b, b));
EXPECT_TRUE (test_evenly_divides(c, c));
EXPECT_FALSE(test_evenly_divides(a, b));
EXPECT_TRUE (test_evenly_divides(a, c));
EXPECT_FALSE(test_evenly_divides(c, a));
auto a = _16{};
auto b = _12{};
auto c = _8{};
EXPECT_TRUE (softly_compatible(a, a));
EXPECT_TRUE (softly_compatible(b, b));
EXPECT_TRUE (softly_compatible(c, c));
EXPECT_FALSE(softly_compatible(a, b));
EXPECT_TRUE (softly_compatible(a, c));
EXPECT_FALSE(softly_compatible(c, a));
auto a0 = make_shape(16);
EXPECT_TRUE (test_evenly_divides(a0, a0));
EXPECT_TRUE (test_evenly_divides(a , a0));
EXPECT_TRUE (test_evenly_divides(a0, a ));
EXPECT_FALSE(test_evenly_divides(c , a0));
EXPECT_TRUE (test_evenly_divides(a0, c ));
EXPECT_FALSE(test_evenly_divides(b , a0));
EXPECT_FALSE(test_evenly_divides(a0, b ));
auto a0 = Shape<_16>{};
EXPECT_TRUE (softly_compatible(a0, a0));
EXPECT_TRUE (softly_compatible(a , a0));
EXPECT_FALSE(softly_compatible(a0, a ));
EXPECT_FALSE(softly_compatible(c , a0));
EXPECT_FALSE(softly_compatible(a0, c ));
EXPECT_FALSE(softly_compatible(b , a0));
EXPECT_FALSE(softly_compatible(a0, b ));
auto a1 = make_shape(2, 8);
EXPECT_TRUE (test_evenly_divides(a1, a1));
EXPECT_FALSE(test_evenly_divides(a , a1));
EXPECT_FALSE(test_evenly_divides(a0, a1));
EXPECT_FALSE(test_evenly_divides(a1, a0));
EXPECT_FALSE(test_evenly_divides(a1, make_shape(2,make_shape(2,4))));
auto a1 = Shape<_2,_8>{};
EXPECT_TRUE (softly_compatible(a1, a1));
EXPECT_TRUE (softly_compatible(a , a1));
EXPECT_FALSE(softly_compatible(a0, a1));
EXPECT_FALSE(softly_compatible(a1, a0));
EXPECT_TRUE (softly_compatible(a1, Shape<_2,Shape<_2,_4>>{}));
auto a2 = make_shape(make_shape(2,8));
EXPECT_TRUE (test_evenly_divides(a2, a2));
EXPECT_FALSE(test_evenly_divides(a , a2));
EXPECT_FALSE(test_evenly_divides(c , a2));
EXPECT_FALSE(test_evenly_divides(a0, a2));
EXPECT_TRUE (test_evenly_divides(a2, a0));
auto a2 = Shape<Shape<_2,_8>>{};
EXPECT_TRUE (softly_compatible(a2, a2));
EXPECT_TRUE (softly_compatible(a , a2));
EXPECT_FALSE(softly_compatible(c , a2));
EXPECT_TRUE (softly_compatible(a0, a2));
EXPECT_FALSE(softly_compatible(a2, a0));
auto a3 = make_shape(make_shape(2,make_shape(4,2)));
EXPECT_TRUE (test_evenly_divides(a3, a3));
EXPECT_FALSE(test_evenly_divides(a , a3));
EXPECT_FALSE(test_evenly_divides(c , a3));
EXPECT_FALSE(test_evenly_divides(a0, a3));
EXPECT_TRUE (test_evenly_divides(a3, a0));
EXPECT_FALSE(test_evenly_divides(a2, a3));
EXPECT_TRUE (test_evenly_divides(a3, a2));
}
auto a3 = Shape<Shape<_2,Shape<_4,_2>>>{};
EXPECT_TRUE (softly_compatible(a3, a3));
EXPECT_TRUE (softly_compatible(a , a3));
EXPECT_FALSE(softly_compatible(c , a3));
EXPECT_TRUE (softly_compatible(a0, a3));
EXPECT_FALSE(softly_compatible(a3, a0));
EXPECT_TRUE (softly_compatible(a2, a3));
EXPECT_FALSE(softly_compatible(a3, a2));
{
auto a = Shape<_32,_64>{};
EXPECT_TRUE (test_evenly_divides(a, Int<128>{}));
EXPECT_TRUE (test_evenly_divides(a, Tile<Layout<_8,_2>, _32>{}));
EXPECT_FALSE(test_evenly_divides(a, Tile<Layout<_8,_3>, _32>{}));
}
}
test/unit/cute/core/inverse_left.cpp
+4 -2
View File
@@ -32,9 +32,11 @@
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <iostream>
#include <cute/layout.hpp>
#include <cute/layout_composed.hpp> // cute::composition
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::composition
#include <cute/tensor.hpp>
using namespace cute;
test/unit/cute/core/inverse_right.cpp
+4 -1
View File
@@ -33,7 +33,10 @@
#include <cutlass/trace.h>
#include <iostream>
#include <cute/layout.hpp>
#include <cute/layout_composed.hpp> // cute::composition
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::composition
#include <cute/tensor.hpp>
using namespace cute;
test/unit/cute/core/math.cpp
+10
View File
@@ -34,6 +34,8 @@
#include <cutlass/trace.h>
#include <cute/numeric/integral_constant.hpp>
#include <cute/numeric/math.hpp>
#include <cute/swizzle.hpp>
#include <cute/swizzle_layout.hpp>
#include <cute/util/type_traits.hpp>
// If cute::gcd returns auto instead of common_type_t<T, U>,
@@ -123,3 +125,11 @@ TEST(CuTe_core, lcm_returns_common_type)
static_assert(int(result) == 1);
}
}
TEST(CuTe_core, max_alignment)
{
{
constexpr auto swizzle = cute::Swizzle<3,4,3>{};
static_assert(cute::max_alignment(swizzle) == 1 << 4);
}
}
test/unit/cute/core/swizzle_layout.cpp
+116
View File
@@ -0,0 +1,116 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <cute/tensor_impl.hpp>
#include <cute/swizzle_layout.hpp>
template <class SwLayout>
void
test_swizzle_2d(SwLayout const& sw_layout)
{
using namespace cute;
auto sw_tensor = make_tensor(counting_iterator<int>{0}, sw_layout);
//print_tensor(sw_tensor);
// Dynamic slicing
for (int i = 0; i < size<0>(sw_tensor); ++i) {
auto sliced_tensor = sw_tensor(i,_);
//printf("sw_tensor(%d,_) => ", int(i)); print(sliced_tensor); printf("\n");
for (int j = 0; j < size<1>(sw_tensor); ++j) {
EXPECT_EQ(sw_tensor(i,j), sliced_tensor(j));
}
}
// Static slicing
cute::for_each(make_int_sequence<size<0>(sw_tensor)>{}, [&] (auto i) {
auto sliced_tensor = sw_tensor(i,_);
//printf("sw_tensor(%d,_) => ", int(i)); print(sliced_tensor); printf("\n");
// If sw_tensor is static, then sliced_tensor should be too
auto sw_tensor_2 = sw_tensor;
static_assert(is_static<decltype(layout(sliced_tensor))>::value || not is_static<decltype(layout(sw_tensor_2))>::value);
cute::for_each(make_int_sequence<size(sliced_tensor)>{}, [&] (auto j) {
EXPECT_EQ(sw_tensor(i,j), sliced_tensor(j));
});
});
// Dynamic slicing
for (int j = 0; j < size<1>(sw_tensor); ++j) {
auto sliced_tensor = sw_tensor(_,j);
//printf("sw_tensor(_,%d) => ", int(j)); print(sliced_tensor); printf("\n");
for (int i = 0; i < size<0>(sw_tensor); ++i) {
EXPECT_EQ(sw_tensor(i,j), sliced_tensor(i));
}
}
// Static slicing
cute::for_each(make_int_sequence<size<1>(sw_tensor)>{}, [&] (auto j) {
auto sliced_tensor = sw_tensor(_,j);
//printf("sw_tensor(_,%d) => ", int(j)); print(sliced_tensor); printf("\n");
// If sw_tensor is static, then sliced_tensor should be too
auto sw_tensor_2 = sw_tensor;
static_assert(is_static<decltype(layout(sliced_tensor))>::value || not is_static<decltype(layout(sw_tensor_2))>::value);
cute::for_each(make_int_sequence<size(sliced_tensor)>{}, [&] (auto i) {
EXPECT_EQ(sw_tensor(i,j), sliced_tensor(i));
});
});
}
TEST(CuTe_core, SwizzleLayout)
{
using namespace cute;
{
auto sw_layout = composition(Swizzle<3,0,3>{},
Layout<Shape <_8,_8>,
Stride<_8,_1>>{});
test_swizzle_2d(sw_layout);
}
{
auto sw_layout = composition(Swizzle<3,0,-3>{},
Layout<Shape <_8,_8>,
Stride<_8,_1>>{});
test_swizzle_2d(sw_layout);
}
{
auto sw_layout = composition(Swizzle<2,1,3>{},
Layout<Shape <Shape < _2,_2,_2>,Shape <_2,_2, _2>>,
Stride<Stride<_32,_2,_8>,Stride<_4,_1,_16>>>{});
test_swizzle_2d(sw_layout);
}
}
test/unit/cute/hopper/cooperative_gemm.cu
+2
View File
@@ -32,6 +32,8 @@
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::compose(cute::Swizzle)
#include "../cooperative_gemm_common.hpp"
test/unit/cute/layout/layout_operator.cu
+3
View File
@@ -41,6 +41,9 @@
// Cute includes
#include <cute/layout.hpp>
#include <cute/int_tuple.hpp>
#include <cute/swizzle.hpp>
#include <cute/layout_composed.hpp>
#include <cute/swizzle_layout.hpp>
using namespace cutlass;
using namespace cute;
test/unit/cute/volta/cooperative_gemm.cu
+2
View File
@@ -32,6 +32,8 @@
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include <cute/swizzle.hpp> // cute::Swizzle
#include <cute/swizzle_layout.hpp> // cute::compose(cute::Swizzle)
#include "../cooperative_gemm_common.hpp"
test/unit/gemm/device/CMakeLists.txt
+27 -1
View File
@@ -42,6 +42,7 @@ add_custom_target(
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
cutlass_test_unit_gemv_device
cutlass_test_unit_gemm_device_tensorop_sm90
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
)
@@ -255,7 +256,7 @@ cutlass_test_unit_add_executable(
gemm_universal_u8t_f16n_f32t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_s8t_f16n_f16t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_u8t_f16n_f16t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_s8t_bf16n_f32t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_u8t_bf16n_f32t_mixed_input_tensor_op_f32_sm80.cu
gemm_universal_s8t_bf16n_bf16t_mixed_input_tensor_op_f32_sm80.cu
@@ -348,6 +349,31 @@ cutlass_test_unit_add_executable(
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm_pingpong.cu
)
# Sparse tests
# Sparse kernels trigger an ICE in gcc 7.5
if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0))
cutlass_test_unit_add_executable(
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
# No batching of source to control compiler memory usage
BATCH_SOURCES ON
BATCH_SIZE 1
sm90_sparse_gemm_s8_s8_s32_tensor_op_s32.cu
sm90_sparse_gemm_f8_f8_f32_tensor_op_f32.cu
sm90_sparse_gemm_f16_f16_f32_tensor_op_f32.cu
sm90_sparse_gemm_tf32_tf32_f32_tensor_op_f32.cu
)
else()
cutlass_test_unit_add_executable(
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
# No batching of source to control compiler memory usage
BATCH_SOURCES ON
BATCH_SIZE 1
)
endif()
# Fused epilogue tests
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_epilogue_fusion_sm90
test/unit/gemm/device/gemm_testbed_3x.hpp
+731 -76
View File
File diff suppressed because it is too large Load Diff
test/unit/gemm/device/gemm_testbed_3x_evt.hpp
+672 -386
View File
File diff suppressed because it is too large Load Diff
test/unit/gemm/device/gemm_testbed_3x_ptr_array.hpp
+26 -25
View File
@@ -76,7 +76,7 @@ enum class ScalarLoc {
ON_DEVICE = 1
};
enum class VectorBeta {
enum class VectorScale {
DISABLED = 0,
ENABLED = 1
};
@@ -556,8 +556,8 @@ struct HostCollectiveDefaultEpilogue {
CheckEquality check_relative_equality = CheckEquality::EXACT;
// Are scalars copied to device memory before kernel launch
ScalarLoc use_device_scalars = ScalarLoc::ON_HOST;
// If per-row scale is enabled and this is true, beta is passed as a host scalar instead of device vector
VectorBeta disable_vector_beta = VectorBeta::DISABLED;
// If per-row scale is enabled and this is disabled, alpha/beta are passed as a host or device scalar instead of device vector
VectorScale vector_scale_mode = VectorScale::DISABLED;
cutlass::Distribution::Kind init_C;
uint64_t seed;
@@ -566,7 +566,7 @@ struct HostCollectiveDefaultEpilogue {
HostCollectiveDefaultEpilogue(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
@@ -850,7 +850,7 @@ struct HostCollectiveEpilogue {
std::vector<cutlass::HostTensor<ElementC, LayoutTagC>> tensors_C;
cutlass::DeviceAllocation<const ElementC *> device_tensors_C;
cutlass::HostTensor<ElementCompute, LayoutTagScalar> norm_constant;
// Outputs
cutlass::HostTensor<ElementAmax, LayoutTagScalar> abs_max_Aux;
cutlass::HostTensor<ElementAmax, LayoutTagScalar> abs_max_D;
@@ -871,8 +871,8 @@ struct HostCollectiveEpilogue {
CheckEquality check_relative_equality = CheckEquality::EXACT;
// Are scalars copied to device memory before kernel launch
ScalarLoc use_device_scalars = ScalarLoc::ON_HOST;
// If per-row scale is enabled and this is true, beta is passed as a host scalar instead of device vector
VectorBeta disable_vector_beta = VectorBeta::DISABLED;
// If per-row scale is enabled and this is disabled, alpha/beta are passed as a host or device scalar instead of device vector
VectorScale vector_scale_mode = VectorScale::DISABLED;
// Random distribution with which to initialize the A/B/C/D/Aux scaling factors
cutlass::Distribution::Kind init_scale = cutlass::Distribution::Uniform;
@@ -885,7 +885,7 @@ struct HostCollectiveEpilogue {
HostCollectiveEpilogue(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
@@ -932,7 +932,7 @@ struct HostCollectiveEpilogue {
if constexpr (IsPerRowScaleEnabled) {
alpha.resize(col_vector_coord);
EXPECT_TRUE(initialize_tensor(alpha.host_view(), init_scale, seed + 2023));
if (disable_vector_beta == VectorBeta::DISABLED) {
if (vector_scale_mode == VectorScale::DISABLED) {
beta.resize(scalar_coord, false);
cutlass::reference::host::TensorFill(beta.host_view(), beta_);
}
@@ -1004,7 +1004,7 @@ struct HostCollectiveEpilogue {
}
stride_Aux = cutlass::make_cute_packed_stride(cutlass::gemm::TagToStrideC_t<LayoutTagAux>{}, cute::make_shape(M, N, 1));
}
static_assert(!IsGroupGemm or (IsGroupGemm and !IsAuxOutEnabled));
if constexpr (IsAuxOutEnabled) {
@@ -1250,9 +1250,10 @@ struct HostCollectiveEpilogue {
else {
fusion_args.alpha = alpha.at(coord_0);
fusion_args.beta = beta.at(coord_0);
fusion_args.alpha_ptr = alpha.device_data();
fusion_args.beta_ptr = beta.device_data(); // if disable_vector_beta is true this is nullptr
fusion_args.alpha_ptr = alpha.device_data();
fusion_args.beta_ptr = beta.device_data();
if constexpr (IsScaleFactorEnabled) {
fusion_args.scale_a = scale_A.at(coord_0);
fusion_args.scale_b = scale_B.at(coord_0);
@@ -1334,9 +1335,9 @@ struct HostCollectiveEpilogue {
return cute::make_tensor(ptr, Aux_layout);
}();
auto Valpha = cute::make_tensor(detail::make_iterator(alpha.host_data()),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cute::make_layout(cute::make_shape(M, N, cute::_1{}), cute::make_stride(cute::_1{}, cute::_0{}, M)));
auto Vbeta = cute::make_tensor(detail::make_iterator(beta.host_data()),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cute::make_layout(cute::make_shape(M, N, cute::_1{}), cute::make_stride(cute::_1{}, cute::_0{}, N)));
cutlass::reference::host::GettEpilogueParams<
ElementScalar,
@@ -1388,7 +1389,7 @@ struct HostCollectiveEpilogue {
if constexpr (IsPerRowScaleEnabled) {
epilogue_params.Valpha = Valpha;
if (disable_vector_beta == VectorBeta::ENABLED) {
if (vector_scale_mode == VectorScale::ENABLED) {
epilogue_params.Vbeta = Vbeta;
}
}
@@ -1442,7 +1443,7 @@ struct TestbedImpl {
TestbedImpl(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
@@ -1450,7 +1451,7 @@ struct TestbedImpl {
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, init_A_, init_B_, seed_)),
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, disable_vector_beta_, init_C_, init_scale_, init_bias_, seed_)) { }
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_C_, init_scale_, init_bias_, seed_)) { }
TestbedImpl(
typename LayoutTagA::Stride stride_factor_A_,
@@ -1459,7 +1460,7 @@ struct TestbedImpl {
typename LayoutTagD::Stride stride_factor_D_,
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_HOST,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
@@ -1467,7 +1468,7 @@ struct TestbedImpl {
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, stride_factor_A_, stride_factor_B_, init_A_, init_B_, seed_)),
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, disable_vector_beta_, init_C_, init_scale_, init_bias_, seed_)) { }
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_C_, init_scale_, init_bias_, seed_)) { }
/// Initializes data structures
bool initialize(ProblemShapeType problem_shapes, ElementScalar alpha_=1.f, ElementScalar beta_=0.f) {
@@ -1501,7 +1502,7 @@ struct TestbedImpl {
file
<< "problem: " << ' ' << M << "x" << N << "x" << K << ", Batch count = " << batch
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
collective_mma_inputs.print_tensors(file, batch);
collective_epilogue.print_tensors(file, batch);
}
@@ -1523,7 +1524,7 @@ struct TestbedImpl {
for (int32_t i = 0; i < L; ++i) {
auto mainloop_params = collective_mma_inputs.to_host_args(problem_shapes, i);
auto epilogue_params = collective_epilogue.to_host_args(problem_shapes, i);
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
passed &= compare_reference(problem_shapes, alpha, beta, i);
@@ -1531,7 +1532,7 @@ struct TestbedImpl {
return passed;
}
/// Determine if the CUDA device is sufficient to run the kernel
/// Determine if the CUDA device is sufficient to run the kernel
bool sufficient() {
//
// Determine SMEM requirements and waive if not satisfied
@@ -1704,14 +1705,14 @@ struct Testbed3x {
Testbed3x(
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
ScalarLoc use_device_scalars_ = ScalarLoc::ON_DEVICE,
VectorBeta disable_vector_beta_ = VectorBeta::DISABLED,
VectorScale vector_scale_mode_ = VectorScale::DISABLED,
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = TestBedImpl::kDefaultSeed)
: impl_(check_relative_equality_, use_device_scalars_, disable_vector_beta_, init_A_, init_B_, init_C_, init_scale_, init_bias_, seed_) {}
: impl_(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_A_, init_B_, init_C_, init_scale_, init_bias_, seed_) {}
/// Executes one test
bool run(
@@ -1734,7 +1735,7 @@ bool TestAll(double alpha = 1.0, double beta = 0.0, CheckEquality check_relative
using ElementScalar = typename Gemm::EpilogueOutputOp::ElementScalar;
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
Testbed3x<Gemm, ActivationFunctor> testbed(check_relative_equality, ScalarLoc::ON_DEVICE, VectorBeta::DISABLED);
Testbed3x<Gemm, ActivationFunctor> testbed(check_relative_equality, ScalarLoc::ON_DEVICE, VectorScale::DISABLED);
int max_alignment = std::max(Gemm::kAlignmentA, Gemm::kAlignmentB);
std::vector<int> problem_size_m = {max_alignment, 512 - 3 * max_alignment};
test/unit/gemm/device/gemm_testbed_3x_tensor_broadcast.hpp
+4 -4
View File
@@ -101,7 +101,7 @@ struct Testbed3xTensorBroadcast {
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
uint64_t seed_ = TestBedImpl::kDefaultSeed
) :
impl_(CheckEquality::EXACT, ScalarLoc::ON_DEVICE, VectorBeta::ENABLED,
impl_(CheckEquality::EXACT, ScalarLoc::ON_DEVICE, VectorScale::ENABLED,
init_A_, init_B_, init_C_, cutlass::Distribution::Uniform, cutlass::Distribution::Uniform, seed_) { }
Testbed3xTensorBroadcast(
@@ -118,7 +118,7 @@ struct Testbed3xTensorBroadcast {
stride_factor_B_,
stride_factor_C_,
stride_factor_D_,
CheckEquality::EXACT, ScalarLoc::ON_HOST, VectorBeta::ENABLED,
CheckEquality::EXACT, ScalarLoc::ON_HOST, VectorScale::ENABLED,
init_A_,
init_B_,
init_C_,
@@ -255,9 +255,9 @@ struct Testbed3xTensorBroadcast {
auto dummy_Aux = cute::make_tensor(static_cast<ElementD*>(nullptr),
cute::make_layout(cute::make_shape(M, N, L), impl_.collective_epilogue.stride_d));
auto dummy_Valpha = cute::make_tensor(static_cast<ElementCompute*>(nullptr),
cute::make_layout(cute::make_shape(M, 1)));
cute::make_layout(cute::make_shape(M, N, 1), cute::make_stride(cute::_1{}, cute::_0{}, M)));
auto dummy_Vbeta = cute::make_tensor(static_cast<ElementCompute*>(nullptr),
cute::make_layout(cute::make_shape(M, 1)));
cute::make_layout(cute::make_shape(M, N, 1), cute::make_stride(cute::_1{}, cute::_0{}, M)));
cutlass::reference::host::GettEpilogueParams<
ElementScalar,
ElementScalar,
test/unit/gemm/device/gemm_universal_bf16t_u8n_bf16t_mixed_input_tensor_op_f32_sm80.cu
+9 -9
View File
@@ -30,7 +30,7 @@
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
@@ -64,14 +64,14 @@ TEST(SM80_Device_GemmUniversal_bf16t_u8n_bf16t_mixed_input_tensor_op_f32, 128x12
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmUniversal<
ElementA,
cutlass::layout::RowMajor,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
ElementA,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
@@ -79,7 +79,7 @@ TEST(SM80_Device_GemmUniversal_bf16t_u8n_bf16t_mixed_input_tensor_op_f32, 128x12
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4, // Stages
8, // AlignmentA
16, // AlignmentB
test/unit/gemm/device/gemm_universal_u8t_bf16n_bf16t_mixed_input_tensor_op_f32_sm80.cu
+9 -9
View File
@@ -30,7 +30,7 @@
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
@@ -64,14 +64,14 @@ TEST(SM80_Device_GemmUniversal_u8t_bf16n_bf16t_mixed_input_tensor_op_f32, 128x12
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmUniversal<
ElementA,
cutlass::layout::RowMajor,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
ElementA,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
@@ -79,7 +79,7 @@ TEST(SM80_Device_GemmUniversal_u8t_bf16n_bf16t_mixed_input_tensor_op_f32, 128x12
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4, // Stages
16, // AlignmentA
8, // AlignmentB
test/unit/gemm/device/sm90_evt_operations.hpp
+151 -97
View File
@@ -37,41 +37,51 @@
//////////////////////////////////////////////////////////////////////////////
/// Host references used for testing
namespace test::gemm::device {
template<class Gemm, class NodeOp, class ...ChildOp>
using HEVT = HostTreeVisitor<Gemm, NodeOp, ChildOp...>;
template<class NodeOp, class ...ChildOp>
using HEVT = HostTreeVisitor<NodeOp, ChildOp...>;
template<class Gemm, class EdgeTuple, class ...Ops>
using HDAG = HostTopoVisitor<Gemm, EdgeTuple, Ops...>;
template<class EdgeTuple, class ...Ops>
using HDAG = HostTopoVisitor<EdgeTuple, Ops...>;
template<class Gemm, class InputTree, class OutputTree, class... AuxOutTrees>
using HST = HostSplitTreeVisitor<Gemm, InputTree, OutputTree, AuxOutTrees...>;
template<class InputTree, class OutputTree, class... AuxOutTrees>
using HST = HostSplitTreeVisitor<InputTree, OutputTree, AuxOutTrees...>;
/// D = alpha * acc + beta * C + AuxLoad
template<class Gemm, class ElementAux, class LayoutAux>
class HostEVTAuxLoad {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using AuxLoadNode = HostAuxLoad<Gemm, false, ElementAux, LayoutAux>;
using TernaryCompute0 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, AuxLoadNode>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, TernaryCompute1>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using AuxLoadNode = HostAuxLoad<ElementAux, LayoutAux, false>;
using TernaryCompute0 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, AuxLoadNode>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, TernaryCompute1>;
};
/// D = alpha * acc + beta * C + per-column bias
template<class Gemm, class ElementBias>
class HostPerColBias {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using RowBroadcastNode = HostRowBroadcast<Gemm, ElementBias>;
using TernaryCompute0 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, RowBroadcastNode>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, TernaryCompute1>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using RowBroadcastNode = HostRowBroadcast<ElementBias>;
using TernaryCompute0 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarAlpha, AccFetchNode, RowBroadcastNode>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, TernaryCompute0>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, TernaryCompute1>;
};
/// D = beta * C + Graph(relu(alpha * acc + aux) + aux)
@@ -79,11 +89,16 @@ public:
template<class Gemm>
class HostEVTDAG {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using AuxLoadNode = HostAuxLoad<Gemm, false, cutlass::half_t, cutlass::layout::RowMajor>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using AuxLoadNode = HostAuxLoad<cutlass::half_t, cutlass::layout::RowMajor, false>;
using DAGNode = HDAG<
Gemm,
float,
cute::tuple<
cute::tuple<>, // 0. alpha
cute::tuple<>, // 1. acc
@@ -95,14 +110,14 @@ public:
ScalarAlpha,
AccFetchNode,
AuxLoadNode,
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostCompute<Gemm, cutlass::epilogue::thread::ReLu>,
HostCompute<Gemm, cutlass::plus>
HostCompute<cutlass::homogeneous_multiply_add>,
HostCompute<cutlass::epilogue::thread::ReLu>,
HostCompute<cutlass::plus>
>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, DAGNode>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, TernaryCompute1>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, DAGNode>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, TernaryCompute1>;
};
/// EVT = alpha * acc + C
@@ -111,19 +126,24 @@ public:
template<class Gemm>
class HostDAGEVT {
public:
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using EVTNode = HEVT<
HostAuxStore<Gemm, false, cutlass::half_t, cutlass::layout::RowMajor>,
HostAuxStore<cutlass::half_t, cutlass::layout::RowMajor, false>,
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 2>,
HostAccumulator<Gemm>,
HostAuxLoad<Gemm, true>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<2>,
HostAccumulator<>,
HostAuxLoad<ElementC, LayoutC, true>
>
>;
using EVTModule = HEVT<
HostAuxStore<Gemm, true>,
HostAuxStore<ElementD, LayoutD, true>,
HDAG<
Gemm,
float,
cute::tuple<
cute::tuple<>, // 0. EVT
cute::tuple<>, // 1. per-row bias
@@ -131,25 +151,30 @@ public:
cute::tuple<cute::_0, cute::_2> // 3. maximum(EVT + per-row bias, EVT)
>,
EVTNode,
HostColBroadcast<Gemm, cutlass::half_t>,
HostCompute<Gemm, cutlass::plus>,
HostCompute<Gemm, cutlass::maximum_with_default_nan_propagation>
HostColBroadcast<cutlass::half_t, cute::Stride<cute::_1,cute::_0,int>>,
HostCompute<cutlass::plus>,
HostCompute<cutlass::maximum_with_default_nan_propagation>
>
>;
};
/// Xreduce(alpha * acc + beta * C)
template<class Gemm, template<class, template <class> class, class> class ReduceOp>
template<class Gemm, class ReduceOp>
class HostReduce {
public:
using ScalarAlpha = HostScalarBroadcast<Gemm, 1>;
using AccFetchNode = HostAccumulator<Gemm>;
using BinaryCompute0 = HEVT<HostCompute<Gemm, cutlass::multiplies>, ScalarAlpha, AccFetchNode>;
using ScalarBeta = HostScalarBroadcast<Gemm, 1>;
using CLoadNode = HostAuxLoad<Gemm, true>;
using TernaryCompute1 = HEVT<HostCompute<Gemm, cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, BinaryCompute0>;
using ReduceNode = HEVT<ReduceOp<Gemm, cutlass::plus, float>, TernaryCompute1>;
using EVTModule = HEVT<HostAuxStore<Gemm, true>, ReduceNode>;
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using ElementD = typename Gemm::GemmKernel::ElementC;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using ScalarAlpha = HostScalarBroadcast<1>;
using AccFetchNode = HostAccumulator<>;
using BinaryCompute0 = HEVT<HostCompute<cutlass::multiplies>, ScalarAlpha, AccFetchNode>;
using ScalarBeta = HostScalarBroadcast<1>;
using CLoadNode = HostAuxLoad<ElementC, LayoutC, true>;
using TernaryCompute1 = HEVT<HostCompute<cutlass::homogeneous_multiply_add>, ScalarBeta, CLoadNode, BinaryCompute0>;
using ReduceNode = HEVT<ReduceOp, TernaryCompute1>;
using EVTModule = HEVT<HostAuxStore<ElementD, LayoutD, true>, ReduceNode>;
};
// Z = scale_a * scale_b * alpha * acc + beta * scale_c * C + per-row bias
@@ -160,25 +185,29 @@ public:
template <class Gemm, template <class> class ActivationFn, class ElementD>
class HostScaledLinCombPerRowBiasEltAct {
public:
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
using EVTModule = HEVT<
HostAuxStore<Gemm, true>,
HostAuxStore<ElementD, LayoutD, true>,
HEVT<
HostCompute<Gemm, cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>, // activation(Z) * scaled_d
HostCompute<cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>, // activation(Z) * scaled_d
HEVT<
HostCompute<Gemm, ActivationFn>, // activation(Z)
HostCompute<ActivationFn>, // activation(Z)
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 2>, // scale_c * beta
HostAuxLoad<Gemm, true>, // C
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 2, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_c * beta
HostAuxLoad<ElementC, LayoutC, true>, // C
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 3>, // scale_a * scale_b * alpha
HostAccumulator<Gemm>,
HostColBroadcast<Gemm, ElementD>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 3, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_a * scale_b * alpha
HostAccumulator<>,
HostColBroadcast<ElementD, cute::Stride<cute::_1,cute::_0,int64_t>>
>
>
>,
HostScalarBroadcast<Gemm, 1> // scale_d
HostScalarBroadcast<1> // scale_d
>
>;
};
@@ -197,45 +226,49 @@ public:
template <class Gemm, template <class> class ActivationFn, class ElementD, class ElementAux = ElementD>
class HostScaledLinCombPerRowBiasEltActAmaxAux {
public:
using ElementC = typename Gemm::GemmKernel::ElementC;
using LayoutC = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideC>;
using LayoutD = cutlass::detail::StrideToLayoutTagC_t<typename Gemm::GemmKernel::StrideD>;
template <typename T>
using amax = cutlass::maximum_absolute_value_reduction<T, true>;
using EVTModuleAuxFp8 = HEVT<
HostAuxStore<Gemm, true>,
HST<Gemm,
HostAuxStore<ElementD, LayoutD, true>,
HST<float,
// Z = scale_a * scale_b * alpha * acc + scale_c * beta * C + per-row bias
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 2>, // scale_c * beta
HostAuxLoad<Gemm, true>, // C
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 2, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_c * beta
HostAuxLoad<ElementC, LayoutC, true>, // C
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 3>, // scale_a * scale_b * alpha
HostAccumulator<Gemm>,
HostColBroadcast<Gemm, ElementD>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 3, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_a * scale_b * alpha
HostAccumulator<>,
HostColBroadcast<ElementD, cute::Stride<cute::_1,cute::_0,int64_t>>
>
>,
// D = activation(Z) * scaled_d, amax_d = max(abs(elements in D))
HEVT<
HostCompute<Gemm, cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HostCompute<cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HEVT<
HostScalarReduce<Gemm, amax, float>,
HostScalarReduce<amax, float>,
HEVT<
HostCompute<Gemm, ActivationFn>, //activation(Z) * scaled_d
HostAccumulator<Gemm> // Z
HostCompute<ActivationFn>, //activation(Z) * scaled_d
HostAccumulator<> // Z
>
>,
HostScalarBroadcast<Gemm, 1> // scale_d
HostScalarBroadcast<1> // scale_d
>,
// Aux = Z * scale_aux, amax_aux = max(abs(elements in Aux))
HEVT<
HostAuxStore<Gemm, false, ElementAux, cutlass::layout::RowMajor>,
HostAuxStore<ElementAux, cutlass::layout::RowMajor, false>,
HEVT<
HostCompute<Gemm, cutlass::multiplies>,
HostCompute<cutlass::multiplies>,
HEVT<
HostScalarReduce<Gemm, amax, float>,
HostAccumulator<Gemm>
HostScalarReduce<amax, float>,
HostAccumulator<>
>,
HostScalarBroadcast<Gemm, 1>
HostScalarBroadcast<1>
>
>
>
@@ -243,32 +276,32 @@ public:
using EVTModuleAuxNotFp8 = HEVT<
// D = activation(Z) * scaled_d, amax_d = max(abs(elements in D))
HostAuxStore<Gemm, true>,
HostAuxStore<ElementD, LayoutD, true>,
HEVT<
HostCompute<Gemm, cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HostCompute<cutlass::epilogue::fusion::detail::ScaleOutOp<ElementD>::template Op>,
HEVT<
HostScalarReduce<Gemm, amax, float>,
HostScalarReduce<amax, float>,
HEVT<
HostCompute<Gemm, ActivationFn>, //activation(Z) * scaled_d
HostCompute<ActivationFn>, //activation(Z) * scaled_d
HEVT<
// Aux = Z
HostAuxStore<Gemm, false, ElementAux, cutlass::layout::RowMajor>,
HostAuxStore<ElementAux, cutlass::layout::RowMajor, false>,
// Z = scale_a * scale_b * alpha * acc + scale_c * beta * C + per-row bias
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 2>, // scale_c * beta
HostAuxLoad<Gemm, true>, // C
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 2, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_c * beta
HostAuxLoad<ElementC, LayoutC, true>, // C
HEVT<
HostCompute<Gemm, cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<Gemm, 1, 3>, // scale_a * scale_b * alpha
HostAccumulator<Gemm>,
HostColBroadcast<Gemm, ElementD>
HostCompute<cutlass::homogeneous_multiply_add>,
HostScalarBroadcast<1, 3, cute::Stride<cute::_0,cute::_0,int64_t>>, // scale_a * scale_b * alpha
HostAccumulator<>,
HostColBroadcast<ElementD, cute::Stride<cute::_1,cute::_0,int64_t>>
>
>
>
>
>,
HostScalarBroadcast<Gemm, 1> // scale_d
HostScalarBroadcast<1> // scale_d
>
>;
@@ -314,6 +347,27 @@ using Sm90LinCombAuxLoad =
>
>;
//////////////////////////////////////////////////////////////////////////////
/// D = alpha * acc + beta * C + AuxLoadNoSmem
template<
class EpilogueDescriptor,
class ElementAux,
class StrideAux,
class ElementOutput,
class ElementCompute,
class ElementScalar = ElementCompute,
FloatRoundStyle RoundStyle = FloatRoundStyle::round_to_nearest
>
using Sm90LinCombAuxLoadNoSmem =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + (alpha * acc + bias)
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
Sm90AuxLoad<0, void, ElementAux, StrideAux, void, void> // aux load
>
>;
//////////////////////////////////////////////////////////////////////////////
/// Example DAG
@@ -386,7 +440,7 @@ using Sm90LinCombDAGEVT =
Sm90SrcFetch<ElementOutput>
>
>,
Sm90ColBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>,
Sm90ColBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias, ElementCompute>,
Sm90Compute<plus, ElementCompute, ElementCompute, RoundStyle>,
Sm90Compute<detail::maximum_with_default_nan_propagation, ElementOutput, ElementCompute, RoundStyle>
>;
@@ -409,7 +463,7 @@ using Sm90LinCombPerColumnBias =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
Sm90RowBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>
Sm90RowBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias, ElementCompute>
>
>;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_aux_load.cu
+54
View File
@@ -118,6 +118,60 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_AuxLoadNoSmemF16_RowMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape_MNK = Shape<_256,_128,_64>;
using ClusterShape_MNK = Shape<_2,_2,_1>;
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
using EpilogueDescriptor = cutlass::epilogue::collective::detail::EpilogueDescriptor<
TileShape_MNK, EpilogueTileType, cutlass::half_t, cutlass::half_t, EpilogueSchedule
>;
using FusionCallbacks = cutlass::epilogue::fusion::Sm90LinCombAuxLoadNoSmem<
EpilogueDescriptor, cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t, float, float>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape_MNK, ClusterShape_MNK,
EpilogueTileType,
float, float,
cutlass::half_t, LayoutC, 8,
cutlass::half_t, LayoutC, 8,
EpilogueSchedule,
FusionCallbacks
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
cutlass::half_t, LayoutA, 8,
cutlass::half_t, LayoutB, 8,
float,
TileShape_MNK, ClusterShape_MNK,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostEVTAuxLoad<
Gemm, cutlass::half_t, cutlass::layout::RowMajor
>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>();
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_AuxLoadF16_ColumnMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_aux_store.cu
+64
View File
@@ -329,6 +329,70 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_VoidC_VoidD_AuxStoreNoSmemF16_RowMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape_MNK = Shape<_256,_128,_64>;
using ClusterShape_MNK = Shape<_2,_2,_1>;
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto;
using EpilogueDescriptor = cutlass::epilogue::collective::detail::EpilogueDescriptor<
TileShape_MNK, EpilogueTileType, cutlass::half_t, cutlass::half_t, EpilogueSchedule
>;
using namespace cutlass::epilogue::fusion;
constexpr auto RoundStyle = cutlass::FloatRoundStyle::round_to_nearest;
constexpr bool has_c = false;
using EVT_D = decltype(test::gemm::device::select_evt_d<cutlass::half_t, float, has_c>());
using AuxStore = Sm90AuxStore<0, void, cutlass::half_t, RoundStyle, cutlass::layout::RowMajor, void, void>;
constexpr auto select_kernel = [](auto has_c, auto has_d) {
using FusionCallbacks =
cute::conditional_t<decltype(has_d){}, EVT_D, Sm90EVT<AuxStore, EVT_D>>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape_MNK, ClusterShape_MNK,
EpilogueTileType,
float, float,
cute::conditional_t<decltype(has_c){}, cutlass::half_t, void>, LayoutC, 8,
cute::conditional_t<decltype(has_d){}, cutlass::half_t, void>, LayoutC, 8,
EpilogueSchedule,
FusionCallbacks
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
cutlass::half_t, LayoutA, 8,
cutlass::half_t, LayoutB, 8,
float,
TileShape_MNK, ClusterShape_MNK,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue>;
return GemmKernel{};
};
using GemmKernel = decltype(select_kernel(cute::C<has_c>{}, cute::C<true>{}));
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
using GemmKernelWithoutD = decltype(select_kernel(cute::C<has_c>{}, cute::C<false>{}));
using GemmWithoutD = cutlass::gemm::device::GemmUniversalAdapter<GemmKernelWithoutD>;
bool passed = test::gemm::device::testEVTAuxStoreWithoutD<Gemm, GemmWithoutD>();
EXPECT_TRUE(passed);
}
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_epilogue, 256x128x64_2x2x1_VoidC_VoidD_AuxStoreF16_ColumnMajor) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_reduce.cu
+3 -3
View File
@@ -102,7 +102,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -148,7 +148,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -194,7 +194,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_pingpong_reduce.cu
+3 -3
View File
@@ -102,7 +102,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostRowReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -148,7 +148,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostColumnReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
@@ -194,7 +194,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce>;
using HostReference = test::gemm::device::HostReduce<Gemm, test::gemm::device::HostScalarReduce<cutlass::plus, float>>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
+126
View File
@@ -119,6 +119,132 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1_ReLu) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule,
cutlass::epilogue::fusion::LinCombEltAct<cutlass::epilogue::thread::ReLu, ElementC, ElementAccumulator>
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
result = TestAll<Gemm>(1.0, 0.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1_silu) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule,
cutlass::epilogue::fusion::LinCombEltAct<cutlass::epilogue::thread::SiLu, ElementC, ElementAccumulator>
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
result = TestAll<Gemm>(1.0, 0.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1_direct_store) {
// A matrix configuration
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm_pingpong.cu
+64 -1
View File
@@ -119,6 +119,69 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm_pingpong, 128x128x64_2x2x1_gelu) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpong; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule,
cutlass::epilogue::fusion::LinCombEltAct<cutlass::epilogue::thread::GELU, ElementC, ElementAccumulator>
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
result = TestAll<Gemm>(1.0, 0.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm_pingpong, 128x128x64_2x2x1_direct_store) {
// A matrix configuration
@@ -181,4 +244,4 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(result);
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array_pingpong.cu
+1 -1
View File
@@ -179,4 +179,4 @@ using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
EXPECT_TRUE(TestAll<Gemm>(1.0, 0.0));
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f16_f16_f32_tensor_op_f32_rs_cluster_warpspecialized_cooperative.cu
+6 -6
View File
@@ -83,12 +83,12 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative, 128x192x64_
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
@@ -159,12 +159,12 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative, 128x192x64_
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
test/unit/gemm/device/sm90_gemm_f32_f32_f32_tensor_op_f32.cu
+3 -3
View File
@@ -186,13 +186,13 @@ TEST(SM90_Device_Gemm_f32t_f32t_f32n_tensor_op_gmma_f32, 128x128x32_1x1x1_cooper
cutlass::detail::TagToStrideA_t<LayoutA>,
float,
cutlass::detail::TagToStrideB_t<LayoutB>,
decltype(cute::make_tiled_mma(cute::SM90_64x64x8_F32TF32TF32_SS_TN{}, Layout<Shape<_2,_1,_1>>{})),
decltype(cute::make_tiled_mma(cute::SM90_64x64x8_F32TF32TF32_SS_TN<>{}, Layout<Shape<_2,_1,_1>>{})),
cute::SM90_TMA_LOAD,
cute::GMMA::Layout_K_SW128_Atom<tfloat32_t>,
GMMA::Layout_K_SW128_Atom<tfloat32_t>,
void,
cute::identity,
cute::SM90_TMA_LOAD,
cute::GMMA::Layout_K_SW128_Atom<tfloat32_t>,
GMMA::Layout_K_SW128_Atom<tfloat32_t>,
void,
cute::identity
>;
test/unit/gemm/device/sm90_gemm_f8_f8_f32_tensor_op_f32_rs_cluster_warpspecialized_cooperative.cu
+6 -6
View File
@@ -83,12 +83,12 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative, 128x128x1
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
@@ -159,12 +159,12 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative, 128x128x1
using KernelScheduleType = cutlass::gemm::KernelTmaWarpSpecializedCooperative;
using AtomLayoutMNK = Layout<Shape<_2,_1,_1>>;
using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::rs_op_selector<
using TiledMma = decltype(cute::make_tiled_mma(GMMA::rs_op_selector<
ElementA, ElementB, ElementAccumulator, TileShape_MNK, GMMA::Major::K, GMMA::Major::K>(), AtomLayoutMNK{}));
using GmemTiledCopyA = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{})));
using GmemTiledCopyB = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{})));
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr cute::GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
static constexpr GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutA>();
static constexpr GMMA::Major GmmaMajorB = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_B<LayoutB>();
using SmemLayoutAtomA = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorA, ElementA,
decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{})), false>());
using SmemLayoutAtomB = decltype(cutlass::gemm::collective::detail::rs_smem_selector<GmmaMajorB, ElementB,
test/unit/gemm/device/sm90_gemm_f8_f8_f8_tensor_op_fp32_evt.cu
+58
View File
@@ -194,4 +194,62 @@ TEST(SM90_Device_Gemm_f8t_f8n_f8t_tensor_op_gmma_f32_persistent_epilogue, 64x128
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
// Z = scale_a * scale_b * alpha * acc + beta * scale_c * C + per-row bias
// if D is fp8
// D = scale_d * filter_negative_zeros(Z)
// else
// D = filter_negative_zeros(Z)
TEST(SM90_Device_Gemm_f8t_f8n_f8t_tensor_op_gmma_f32_persistent_epilogue, 64x128x128_1x1x1_ScaledLinCombPerRowBiasEltFilter) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape_MNK = Shape<_64,_128,_128>;
using ClusterShape_MNK = Shape<_1,_1,_1>;
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
using FusionCallbacks = cutlass::epilogue::fusion::Sm90ScaledLinCombPerRowBiasEltAct<
TileShape_MNK, // CtaTileShapeMNK
cutlass::epilogue::thread::ElementwiseFilter, // ActivationFn
cutlass::float_e4m3_t, // ElementOutput
float, // ElementCompute
cutlass::float_e4m3_t // ElementBias
>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape_MNK, ClusterShape_MNK,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
cutlass::float_e4m3_t, LayoutC, 16,
cutlass::float_e4m3_t, LayoutC, 16,
EpilogueSchedule,
FusionCallbacks
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
cutlass::float_e4m3_t, LayoutA, 16,
cutlass::float_e4m3_t, LayoutB, 16,
float,
TileShape_MNK, ClusterShape_MNK,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// Host reference
using HostReference = test::gemm::device::HostScaledLinCombPerRowBiasEltAct<
Gemm, cutlass::epilogue::thread::ElementwiseFilter, cutlass::float_e4m3_t
>;
bool passed = test::gemm::device::TestAllEVT<Gemm, HostReference>(true);
EXPECT_TRUE(passed);
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_stream_k_scheduler.cu
+1 -1
View File
@@ -104,7 +104,7 @@ test_scheduler(
typename Scheduler::Arguments args{};
// Set up the grid for the problem
dim3 grid = Scheduler::get_grid_shape(problem_shape_mnkl, tile_shape, cluster_shape, hw_info, args);
dim3 grid = Scheduler::get_grid_shape(params, problem_shape_mnkl, tile_shape, cluster_shape, hw_info, args);
auto print_info = [&]() {
std::cout << "Failed with problem size "
test/unit/gemm/device/sm90_sparse_gemm_f16_f16_f32_tensor_op_f32.cu
+255
View File
@@ -0,0 +1,255 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f32t_tensorop_f32, 128x128x64_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f32t_tensorop_f32, 128x128x64_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f32t_tensorop_f32, 128x128x64_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpong
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_bf16t_bf16n_f32t_tensorop_f32, 128x128x128_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::bfloat16_t, LayoutA, 16,
cutlass::bfloat16_t, LayoutB, 8,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_f16t_f16n_f16t_tensorop_f16, 128x128x32_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_32>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
half_t, half_t,
half_t, LayoutC, 4,
half_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::half_t, LayoutA, 16,
cutlass::half_t, LayoutB, 8,
half_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/sm90_sparse_gemm_f8_f8_f32_tensor_op_f32.cu
+216
View File
@@ -0,0 +1,216 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x128_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x128_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x256_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_256>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_e4m3t_e5m2n_f32t_tensorop_f32, 128x128x64_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
cutlass::float_e4m3_t, LayoutA, 32,
cutlass::float_e5m2_t, LayoutB, 16,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/sm90_sparse_gemm_s8_s8_s32_tensor_op_s32.cu
+216
View File
@@ -0,0 +1,216 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x128_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x128_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_128>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x256_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_256>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_s8t_s8n_s32t_tensorop_s32, 128x128x64_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
int32_t, int32_t,
int32_t, LayoutC, 4,
int32_t, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
int8_t, LayoutA, 32,
int8_t, LayoutB, 16,
int32_t,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpong
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/sm90_sparse_gemm_tf32_tf32_f32_tensor_op_f32.cu
+216
View File
@@ -0,0 +1,216 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/arch/mma_sm90.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/dispatch_policy.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
TEST(SM90_Device_Sparse_Gemm_tf16t_tf16n_f32t_tensorop_f32, 128x128x64_1x1x1) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::collective::KernelScheduleAuto
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_tf16t_tf16n_f32t_tensorop_f32, 128x128x64_1x1x1_warpspecialized) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_64>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecialized
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_tf32t_tf32n_f32t_tensorop_f32, 128x128x32_1x2x1_cooperative) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_32>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedCooperative
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Sparse_Gemm_tf32t_tf32n_f32t_tensorop_f32, 128x128x16_2x1x1_pingpong) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
using TileShape = Shape<_128,_128,_16>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecialized
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
float, LayoutA, 8,
float, LayoutB, 4,
float,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::gemm::KernelTmaWarpSpecializedPingpong
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0, CheckEquality::EXACT);
EXPECT_TRUE(result);
}
#endif // #if defined(CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED)
test/unit/gemm/device/testbed.h
+68 -28
View File
@@ -208,15 +208,17 @@ struct Testbed {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
if (tensor_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
if (reference_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
if (tensor_D.size() > 1) {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0)
<< "tensor_D (size " << tensor_D.size() << ") has nonpositive norm";
}
if (reference_D.size() > 1) {
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0)
<< "reference_D (size " << reference_D.size() << ") has nonpositive norm";
}
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
EXPECT_TRUE(passed);
EXPECT_TRUE(passed) << "reference_D does not equal tensor_D";
if (!passed) {
@@ -369,9 +371,11 @@ struct Testbed {
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
EXPECT_TRUE(status == cutlass::Status::kSuccess)
<< "gemm_op.initialize returned with error " << to_string(status)
<< ", indicating that this test is not supported. Last CUDA error: "
<< cudaGetErrorString(cudaGetLastError());
if (status != cutlass::Status::kSuccess) {
cudaError_t error = cudaGetLastError();
std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
return true;
}
@@ -379,19 +383,27 @@ struct Testbed {
// Run the GEMM
//
status = gemm_op();
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
try {
status = gemm_op();
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "gemm_op() threw a std::exception: " << e.what();
throw;
}
catch (...) {
EXPECT_TRUE(false) << "gemm_op() threw an exception of unknown type";
throw;
}
EXPECT_TRUE(status == cutlass::Status::kSuccess)
<< "gemm_op failed with error " << to_string(status);
//
// Verify
//
bool passed = this->verify(problem_size, alpha, beta);
if (!passed) {
std::cout << "Error with split_k_slices = " << split_k_slices << ", alpha: " << alpha << std::endl;
}
EXPECT_TRUE(passed) << "Error: split_k_slices = " << split_k_slices
<< ", alpha: " << alpha;
return passed;
}
@@ -470,12 +482,26 @@ bool TestAllGemmBasic(
for (auto beta : problem_beta) {
cutlass::gemm::GemmCoord problem_size(m, n, k);
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
try {
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "TestAllGemmBasic: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: " << e.what();
throw;
}
catch (...) {
EXPECT_TRUE(false) << "TestAllGemmBasic: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: (unknown)";
throw;
}
if (!passed) {
return false;
@@ -570,12 +596,26 @@ bool TestGemmPerf(int iterations = 1) {
cutlass::gemm::GemmCoord problem_size(m, n, k);
for (int i = 0; i < iterations; i++){
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
try {
passed = testbed.run(
problem_size,
split_k,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "TestGemmPerf: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: " << e.what();
throw;
}
catch (...) {
EXPECT_TRUE(false) << "TestGemmPerf: testbed.run threw an "
"exception {alpha: " << alpha << ", beta: " << beta << ", m: "
<< m << ", n: " << n << ", k: " << k << "}: (unknown)";
throw;
}
}
if (!passed) {
test/unit/gemm/device/testbed_gemm_with_broadcast.h
+1 -1
View File
@@ -161,7 +161,7 @@ struct TestbedGemmWithBroadcast {
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_max = 1;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
test/unit/gemm/device/testbed_gemm_with_reduction.h
+1 -1
View File
@@ -143,7 +143,7 @@ struct TestbedGemmWithReduction {
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_max = 1;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
test/unit/gemm/device/testbed_universal.h
+1
View File
@@ -201,6 +201,7 @@ struct TestbedUniversal {
if (!passed) {
/*
std::stringstream fname;
fname << "error_Gemm_device_"
test/unit/gemm/threadblock/mma_multistage.cu
-14
View File
@@ -3006,7 +3006,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3041,7 +3040,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3076,7 +3074,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3111,7 +3108,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3146,7 +3142,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3181,7 +3176,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3216,7 +3210,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3251,7 +3244,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3286,7 +3278,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3321,7 +3312,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3356,7 +3346,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3391,7 +3380,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3426,7 +3414,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_crosswise,
@@ -3461,7 +3448,6 @@ TEST(SM80_gemm_threadblock_crosswise,
problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_gemm_threadblock_congruous,
tensor_op_64x64x16_32x64x16_8x8x4_3stage) {
test/unit/gemm/warp/gemm_sm80.cu
-209
View File
@@ -1357,215 +1357,6 @@ TEST(SM80_warp_gemm_tensor_op_crosswise_i4, 128x128x256_16x16x256_16x8x64) {
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_64x64x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 64, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_64x32x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 32, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_32x32x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 32, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_32x16x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 16, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x512_16x16x512_16x8x256) {
using Shape = cutlass::gemm::GemmShape<16, 16, 512>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 512>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 512> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_64x64x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 64, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_64x32x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<64, 32, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_32x32x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 32, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_32x16x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<32, 16, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_crosswise_b1, 128x128x1024_16x16x1024_16x8x256) {
using Shape = cutlass::gemm::GemmShape<16, 16, 1024>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 256>;
using Element = cutlass::uint1b_t;
using ElementC = int;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 1024>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAdd>::Type;
test::gemm::warp::Testbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 1024> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_congruous_f64, 16x16x4_16x16x4_8x8x4) {
using Shape = cutlass::gemm::GemmShape<16, 16, 4>;
test/unit/transform/device/CMakeLists.txt
+58
View File
@@ -0,0 +1,58 @@
# Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Compress Kernel
#
add_custom_target(
cutlass_test_unit_sm90_structured_sparse_gemm_compressor
DEPENDS
cutlass_test_unit_sm90_structured_sparse_gemm_compressor_f32
cutlass_test_unit_sm90_structured_sparse_gemm_compressor_f16
cutlass_test_unit_sm90_structured_sparse_gemm_compressor_f8
)
cutlass_test_unit_add_executable(
cutlass_test_unit_sm90_structured_sparse_gemm_compressor_f32
sm90_sparse_gemm_compressor_f32.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_sm90_structured_sparse_gemm_compressor_f16
sm90_sparse_gemm_compressor_f16.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_sm90_structured_sparse_gemm_compressor_f8
sm90_sparse_gemm_compressor_f8.cu
)
test/unit/transform/device/sm90_sparse_gemm_compressor_f16.cu
+95
View File
@@ -0,0 +1,95 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cute/atom/mma_traits_sm90_gmma.hpp" // cute::GMMA::Major
#include "cutlass/arch/config.h" // CUTLASS_ARCH_MMA_SM90_SUPPORTED
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp" // StructuredSparseCompressor
#include "cutlass/transform/device/transform_universal_adapter.hpp" // TransformUniversalAdapter
#include "cutlass/gemm/collective/builders/sm90_common.inl" // gmma_ss_tag_to_major_A
#include "cutlass/gemm/collective/builders/sm90_sparse_config.inl" // Sm90GemmSparseConfig
#include "testbed_sparse_gemm_compressor.hpp" // TestbedSparseGemmCompressor
///////////////////////////////////////////////////////////////////////////////////////////////////
// * Test Plan
// ElementA : fp16
// LayoutA : row / col
// Gemm : 1x 2x 3x multiplier of alignment requirement. corner case that smaller than alignment requirement
///////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
TEST(SM90_Structured_Sparse_Gemm_Compressor_Device, f16_t)
{
// Test Settings
using ElementA = cutlass::half_t;
using LayoutATag = cutlass::layout::RowMajor;
// Deduct From Test Setting
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutATag>();
using ElementAMma = cute::sparse_elem<2, ElementA>;
using ElementEMma = cute::sparse_elem<8, uint8_t>;
using SparseConfig = cutlass::Sm90GemmSparseConfig<ElementAMma, GmmaMajorA, ElementEMma, cute::Int<32>>;
using CompressorKernel = cutlass::transform::kernel::
StructuredSparseCompressor<cute::Shape<int, int, int, int>, ElementA, LayoutATag, SparseConfig, cutlass::arch::Sm90>;
using Compressor = cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
// Test Bed
test::transform::device::TestbedSparseGemmCompressor<Compressor> testbed;
EXPECT_TRUE(testbed.run_auto());
}
TEST(SM90_Structured_Sparse_Gemm_Compressor_Device, f16_n)
{
// Test Settings
using ElementA = cutlass::bfloat16_t;
using LayoutATag = cutlass::layout::ColumnMajor;
// Deduct From Test Setting
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutATag>();
using ElementAMma = cute::sparse_elem<2, ElementA>;
using ElementEMma = cute::sparse_elem<8, uint8_t>;
using SparseConfig = cutlass::Sm90GemmSparseConfig<ElementAMma, GmmaMajorA, ElementEMma, cute::Int<64>>;
using CompressorKernel = cutlass::transform::kernel::
StructuredSparseCompressor<cute::Shape<int, int, int, int>, ElementA, LayoutATag, SparseConfig, cutlass::arch::Sm90>;
using Compressor = cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
// Test Bed
test::transform::device::TestbedSparseGemmCompressor<Compressor> testbed;
EXPECT_TRUE(testbed.run_auto());
}
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/transform/device/sm90_sparse_gemm_compressor_f32.cu
+95
View File
@@ -0,0 +1,95 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cute/atom/mma_traits_sm90_gmma.hpp" // cute::GMMA::Major
#include "cutlass/arch/config.h" // CUTLASS_ARCH_MMA_SM90_SUPPORTED
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp" // StructuredSparseCompressor
#include "cutlass/transform/device/transform_universal_adapter.hpp" // TransformUniversalAdapter
#include "cutlass/gemm/collective/builders/sm90_common.inl" // gmma_ss_tag_to_major_A
#include "cutlass/gemm/collective/builders/sm90_sparse_config.inl" // Sm90GemmSparseConfig
#include "testbed_sparse_gemm_compressor.hpp" // TestbedSparseGemmCompressor
///////////////////////////////////////////////////////////////////////////////////////////////////
// * Test Plan
// ElementA : fp32
// LayoutA : row / col
// Gemm : 1x 2x 3x multiplier of alignment requirement. corner case that smaller than alignment requirement
///////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
TEST(SM90_Structured_Sparse_Gemm_Compressor_Device, f32_t)
{
// Test Settings
using ElementA = float;
using LayoutATag = cutlass::layout::RowMajor;
// Deduct From Test Setting
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutATag>();
using ElementAMma = cute::sparse_elem<2, ElementA>;
using ElementEMma = cute::sparse_elem<4, uint8_t>;
using SparseConfig = cutlass::Sm90GemmSparseConfig<ElementAMma, GmmaMajorA, ElementEMma, cute::Int<16>>;
using CompressorKernel = cutlass::transform::kernel::
StructuredSparseCompressor<cute::Shape<int, int, int, int>, ElementA, LayoutATag, SparseConfig, cutlass::arch::Sm90>;
using Compressor = cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
// Test Bed
test::transform::device::TestbedSparseGemmCompressor<Compressor> testbed;
EXPECT_TRUE(testbed.run_auto());
}
TEST(SM90_Structured_Sparse_Gemm_Compressor_Device, f32_n)
{
// Test Settings
using ElementA = cutlass::tfloat32_t;
using LayoutATag = cutlass::layout::ColumnMajor;
// Deduct From Test Setting
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutATag>();
using ElementAMma = cute::sparse_elem<2, ElementA>;
using ElementEMma = cute::sparse_elem<4, uint8_t>;
using SparseConfig = cutlass::Sm90GemmSparseConfig<ElementAMma, GmmaMajorA, ElementEMma, cute::Int<32>>;
using CompressorKernel = cutlass::transform::kernel::
StructuredSparseCompressor<cute::Shape<int, int, int, int>, ElementA, LayoutATag, SparseConfig, cutlass::arch::Sm90>;
using Compressor = cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
// Test Bed
test::transform::device::TestbedSparseGemmCompressor<Compressor> testbed;
EXPECT_TRUE(testbed.run_auto());
}
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/transform/device/sm90_sparse_gemm_compressor_f8.cu
+95
View File
@@ -0,0 +1,95 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cute/atom/mma_traits_sm90_gmma.hpp" // cute::GMMA::Major
#include "cutlass/arch/config.h" // CUTLASS_ARCH_MMA_SM90_SUPPORTED
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp" // StructuredSparseCompressor
#include "cutlass/transform/device/transform_universal_adapter.hpp" // TransformUniversalAdapter
#include "cutlass/gemm/collective/builders/sm90_common.inl" // gmma_ss_tag_to_major_A
#include "cutlass/gemm/collective/builders/sm90_sparse_config.inl" // Sm90GemmSparseConfig
#include "testbed_sparse_gemm_compressor.hpp" // TestbedSparseGemmCompressor
///////////////////////////////////////////////////////////////////////////////////////////////////
// * Test Plan
// ElementA : fp8
// LayoutA : row / col
// Gemm : 1x 2x 3x multiplier of alignment requirement. corner case that smaller than alignment requirement
///////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
TEST(SM90_Structured_Sparse_Gemm_Compressor_Device, f8_t)
{
// Test Settings
using ElementA = cutlass::float_e4m3_t;
using LayoutATag = cutlass::layout::RowMajor;
// Deduct From Test Setting
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutATag>();
using ElementAMma = cute::sparse_elem<2, ElementA>;
using ElementEMma = cute::sparse_elem<8, uint8_t>;
using SparseConfig = cutlass::Sm90GemmSparseConfig<ElementAMma, GmmaMajorA, ElementEMma, cute::Int<64>>;
using CompressorKernel = cutlass::transform::kernel::
StructuredSparseCompressor<cute::Shape<int, int, int, int>, ElementA, LayoutATag, SparseConfig, cutlass::arch::Sm90>;
using Compressor = cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
// Test Bed
test::transform::device::TestbedSparseGemmCompressor<Compressor> testbed;
EXPECT_TRUE(testbed.run_auto());
}
TEST(SM90_Structured_Sparse_Gemm_Compressor_Device, f8_n)
{
// Test Settings
using ElementA = cutlass::float_e5m2_t;
using LayoutATag = cutlass::layout::ColumnMajor;
// Deduct From Test Setting
static constexpr cute::GMMA::Major GmmaMajorA = cutlass::gemm::collective::detail::gmma_rs_tag_to_major_A<LayoutATag>();
using ElementAMma = cute::sparse_elem<2, ElementA>;
using ElementEMma = cute::sparse_elem<8, uint8_t>;
using SparseConfig = cutlass::Sm90GemmSparseConfig<ElementAMma, GmmaMajorA, ElementEMma, cute::Int<64>>;
using CompressorKernel = cutlass::transform::kernel::
StructuredSparseCompressor<cute::Shape<int, int, int, int>, ElementA, LayoutATag, SparseConfig, cutlass::arch::Sm90>;
using Compressor = cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
// Test Bed
test::transform::device::TestbedSparseGemmCompressor<Compressor> testbed;
EXPECT_TRUE(testbed.run_auto());
}
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/transform/device/sm90_sparse_gemm_compressor_legacy.hpp
+480
View File
@@ -0,0 +1,480 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Compress utils specific for SM90 structure sparse kernels
*/
#pragma once
#include <algorithm> // std::fill
#include <array> // std::array
#include <cstdio>
#include <random> // std::mt19937
#include "cute/container/bit_field.hpp" // cute::bit_field
#include "cute/numeric/numeric_types.hpp" // cute::sizeof_bits_v
#include "cute/tensor.hpp" // cute::Tensor, cute::make_tensor, cute::print_tensor
#include "cutlass/arch/arch.h" // cutlass::arch::Sm90
#include "cutlass/cutlass.h" // cutlass::Status
#include "cutlass/detail/layout.hpp" // cutlass::TagToStrideA_t
#include "cutlass/fast_math.h" // cutlass::ceil_div, cutlass::round_up
#include "cutlass/kernel_hardware_info.h" // cutlass::KernelHardwareInfo
#include "cutlass/util/packed_stride.hpp" // cutlass::make_cute_packed_stride
#include "cutlass/numeric_size.h" // cutlass::bits_to_bytes
#include "cutlass/cuda_host_adapter.hpp" // cutlass::CudaHostAdapter
namespace cutlass
{
namespace transform
{
namespace kernel
{
using namespace cute;
namespace detail {
template<typename T>
CUTLASS_HOST_DEVICE
static uint8_t
encode_in_chunk_idx_legacy(int in_chunk_idx){
if (sizeof(T) == 4) {
return in_chunk_idx == 0 ? 0b0100 : 0b1110;
}
else {
uint8_t res = 0;
if (in_chunk_idx == 0) {
res = 0b00;
}
else if (in_chunk_idx == 1) {
res = 0b01;
}
else if (in_chunk_idx == 2) {
res = 0b10;
}
else {
res = 0b11;
}
return res;
}
}
template <
class SparseConfig,
class EngineA,
class LayoutA,
class EngineAc,
class LayoutAc
>
CUTLASS_HOST_DEVICE
static void
compress_two_chunks_legacy(
Tensor<EngineA, LayoutA> tensorA,
Tensor<EngineAc, LayoutAc> tensorAc,
uint8_t& meta_two_chunk,
int effective_elems) {
using ElementA = typename EngineAc::value_type;
static constexpr int LogicalElemsAPerChunk = typename SparseConfig::LogicalElemsAPerChunk{};
static constexpr int PhysicalElemsAPerChunk = typename SparseConfig::PhysicalElemsAPerChunk{};
static constexpr int ElemsARawPerElementAMmaRaw = typename SparseConfig::ElemsARawPerElementAMmaRaw{};
static constexpr int ElementEBitsPerElementAMma = typename SparseConfig::ElementEBitsPerElementAMma{};
static constexpr int LogicalSubChunk = ceil_div(LogicalElemsAPerChunk, ElemsARawPerElementAMmaRaw);
static constexpr int PhysicalSubChunk = ceil_div(PhysicalElemsAPerChunk, ElemsARawPerElementAMmaRaw);
/*
Legal metadata chunk in SM90
Index Bin HEX
0, 1 0b0100 4
1, 2 0b1001 9
2, 3 0b1110 E
0, 2 0b1000 8
1, 3 0b1101 D
0, 3 0b1100 C
2, 1 0b0110 6 (Not used)
-----------------------------------
TF32
0 0b0100 4
1 0b1110 E
*/
if (effective_elems <= 0) {
return;
}
// initialize
// 0 is the initial value for this function while 0x44 is the initial value for hardware.
meta_two_chunk = 0;
for (int chunk_idx = 0; chunk_idx < 2; ++chunk_idx) {
// If Only One Chunk within this Two Chunk
if ( effective_elems <= chunk_idx * ElemsARawPerElementAMmaRaw * LogicalSubChunk ) {
break;
}
/// init result;
int non_zero_cnt = 0;
int32_t nnz_chunk_idx[PhysicalSubChunk] = { 0 };
ElementA Ac_chunk[PhysicalSubChunk][ElemsARawPerElementAMmaRaw] = { ElementA{0} };
for (int subchunk_idx = 0; subchunk_idx < LogicalSubChunk; ++subchunk_idx) {
bool is_nz = true;
ElementA subchunk_elems[ElemsARawPerElementAMmaRaw] = { ElementA{0} };
/// Check if subchunk is non-zero
for(int elem_idx = 0; elem_idx < ElemsARawPerElementAMmaRaw; elem_idx++) {
int offset = chunk_idx * LogicalElemsAPerChunk + subchunk_idx * ElemsARawPerElementAMmaRaw + elem_idx;
subchunk_elems[elem_idx] = offset < effective_elems ? tensorA(offset) : ElementA(0);
if (subchunk_elems[elem_idx] != ElementA(0)) {
if (non_zero_cnt >= PhysicalSubChunk) {
#ifdef __CUDA_ARCH__
asm volatile ("brkpt;\n" ::);
#else
throw std::runtime_error("Found extra non-zero elements in a chunk!\n");
#endif
}
is_nz = false;
}
}
/// There is non-zero element in the subchunk
if(!is_nz) {
nnz_chunk_idx[non_zero_cnt] = subchunk_idx;
memcpy(Ac_chunk[non_zero_cnt], subchunk_elems, sizeof(ElementA) * ElemsARawPerElementAMmaRaw);
non_zero_cnt++;
}
}
/*
Special cases
nnz == 1 and non-tf32 and nnz_idx = 3
*/
ElementA elementA_zeros[ElemsARawPerElementAMmaRaw] = { ElementA{0} };
if constexpr (sizeof_bits_v<ElementA> < 32) {
if (non_zero_cnt == 1 && nnz_chunk_idx[0] == 3) {
memcpy(Ac_chunk[1], Ac_chunk[0], sizeof(ElementA) * ElemsARawPerElementAMmaRaw);
memcpy(Ac_chunk[0], elementA_zeros, sizeof(ElementA) * ElemsARawPerElementAMmaRaw);
nnz_chunk_idx[1] = 3;
nnz_chunk_idx[0] = 0;
}
else if (non_zero_cnt == 1) {
memcpy(Ac_chunk[1], elementA_zeros, sizeof(ElementA) * ElemsARawPerElementAMmaRaw);
nnz_chunk_idx[1] = 3;
}
}
/// Setup metadata
uint8_t meta_chunk = 0;
for (int i = 0; i < PhysicalSubChunk; i++) {
meta_chunk = static_cast<uint8_t>(meta_chunk | (encode_in_chunk_idx_legacy<ElementA>(nnz_chunk_idx[i]) << (i * ElementEBitsPerElementAMma)));
for(int j = 0; j < ElemsARawPerElementAMmaRaw; j++) {
tensorAc(chunk_idx * PhysicalElemsAPerChunk + i * ElemsARawPerElementAMmaRaw + j) = Ac_chunk[i][j];
}
}
meta_two_chunk = uint8_t(meta_two_chunk | (meta_chunk << (chunk_idx * _4{})));
}
}
}
template<
class ProblemShape_,
class ElementA_,
class LayoutATag_,
class SparseConfig_
>
class SM90StructuredSparseCompressorLegacy {
public:
using SparseConfig = SparseConfig_;
using ProblemShape = ProblemShape_;
// * EltA
using ElementA = ElementA_;
using ElementAUint = cute::uint_bit_t<cute::sizeof_bits_v<ElementA>>;
static constexpr bool IsRuntimeDataTypeA = cute::is_same_v<ElementA, cutlass::type_erased_dynamic_float8_t> ||
cute::is_same_v<ElementA, cutlass::type_erased_dynamic_float6_t> ||
cute::is_same_v<ElementA, cutlass::type_erased_dynamic_float4_t>;
using ArrayElementA = cute::conditional_t<IsRuntimeDataTypeA,
cute::uint_bit_t<cute::sizeof_bits_v<ElementA>>,
ElementA>;
using ElementAMma = typename SparseConfig::ElementAMma;
using ElementAMmaRaw = typename SparseConfig::ElementAMmaRaw;
using ElementASparsity = typename SparseConfig::ElementASparsity;
using ElementAMmaSparsity = typename SparseConfig::ElementAMmaSparsity;
using LayoutATag = LayoutATag_;
using LayoutA = LayoutATag;
using StrideA = cutlass::gemm::TagToStrideA_t<LayoutATag>;
// * EltE
using ElementEMma = typename SparseConfig::ElementEMma;
using ElementEMmaRaw = typename SparseConfig::ElementEMmaRaw;
using ElementEMmaSparsity = typename SparseConfig::ElementEMmaSparsity;
// * AtomE
using TensorEAtom = typename SparseConfig::TensorEAtom;
using TensorEAtomK = typename SparseConfig::TensorEAtomK;
using TensorEAtomM = typename SparseConfig::TensorEAtomM;
static constexpr int ElemsARawPerElementAMmaRaw = typename SparseConfig::ElemsARawPerElementAMmaRaw{};
static constexpr int LogicalElemsAPerChunk = typename SparseConfig::LogicalElemsAPerChunk{};
static constexpr int PhysicalElemsAPerChunk = typename SparseConfig::PhysicalElemsAPerChunk{};
static constexpr int LogicalElemsAMmaRawPerChunk = cutlass::ceil_div(LogicalElemsAPerChunk, ElemsARawPerElementAMmaRaw);
static constexpr int PhysicalElemsAMmaRawPerChunk = cutlass::ceil_div(PhysicalElemsAPerChunk, ElemsARawPerElementAMmaRaw);
// * Alignment
static constexpr int TensorEAlignmentM = typename SparseConfig::TensorEAlignmentM{};
static constexpr int TensorEAlignmentK = typename SparseConfig::TensorEAlignmentK{};
static constexpr int TensorAAlignmentK = typename SparseConfig::TensorAAlignmentK{};
static constexpr int TensorAAlignmentM = typename SparseConfig::TensorAAlignmentM{};
// Required by `device_kernel`
static constexpr int MaxThreadsPerBlock = 1;
static constexpr int MinBlocksPerMultiprocessor = 1;
using ArchTag = arch::Sm90;
struct SharedStorage {
/* empty, no smem needed */
};
static constexpr int SharedStorageSize = sizeof(SharedStorage);
struct TransformArguments {
ArrayElementA const* ptr_A{nullptr};
StrideA dA{};
ArrayElementA* ptr_ACompress{nullptr};
ElementEMmaRaw* ptr_E{nullptr};
};
using TransformParams = TransformArguments;
struct Arguments {
ProblemShape problem_shape{};
TransformArguments transform{};
KernelHardwareInfo hw_info{};
};
struct Params {
ProblemShape problem_shape{};
TransformParams transform{};
KernelHardwareInfo hw_info{};
void* workspace = nullptr;
};
static Params
to_underlying_arguments(Arguments & args, void* workspace) {
return Params{{args.problem_shape},
{args.transform.ptr_A, args.transform.dA, args.transform.ptr_ACompress, args.transform.ptr_E},
{args.hw_info},
workspace};
}
static Status
can_implement(Arguments const& args) {
auto [M, N, K, L] = args.problem_shape;
if (K % LogicalElemsAPerChunk != 0) {
CUTLASS_TRACE_HOST("SM90 Sparse Compressor CAN NOT IMPLEMENT: GemmK not multiplier of logical chunk size\n");
return Status::kErrorInvalidProblem;
}
return Status::kSuccess;
}
static size_t
get_workspace_size(Arguments const& args) {
auto problem = args.problem_shape;
const int m = cute::size<0>(problem);
const int k = cute::size<2>(problem);
const int l = cute::size<3>(problem);
const int metadata_k = round_up(k, TensorEAlignmentK);
const int metadata_m = round_up(m, TensorEAlignmentM);
const int metadata_bytes = metadata_m * metadata_k / ElementEMmaSparsity{} * l;
return metadata_bytes;
}
static Status
initialize_workspace(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr,
CudaHostAdapter *cuda_adapter = nullptr) {
cudaError_t cuda_error;
auto workspace_size = get_workspace_size(args);
if (workspace_size == 0) {
return Status::kSuccess;
} else if (workspace == nullptr) {
return Status::kErrorInternal;
}
cudaPointerAttributes attri;
cuda_error = cudaPointerGetAttributes(&attri, workspace);
if (cuda_error != cudaSuccess) {
return Status::kErrorInternal;
}
if ( attri.type == cudaMemoryTypeDevice ) {
#if defined(CUTLASS_ENABLE_CUDA_HOST_ADAPTER) && CUTLASS_ENABLE_CUDA_HOST_ADAPTER
CUTLASS_ASSERT(cuda_adapter);
if (Status::kSuccess != cuda_adapter->memsetDevice(workspace, static_cast<uint8_t>(0), workspace_size, stream)) {
return Status::kErrorInternal;
}
#else
cudaMemsetAsync(workspace, 0, workspace_size, stream);
cuda_error = cudaGetLastError();
if (cuda_error != cudaSuccess) {
return Status::kErrorInternal;
}
#endif
} else {
memset(workspace, 0, workspace_size);
}
return Status::kSuccess;
}
static dim3
get_grid_shape(Params const& params) {
return dim3(1, 1, 1);
}
static dim3
get_block_shape() {
return dim3(1, 1, 1);
}
CUTE_HOST_DEVICE
void
operator()(Params params, char* smem_buf = nullptr) {
run(params, smem_buf);
}
CUTE_HOST_DEVICE
static void
run(Params params, char* smem_buf = nullptr) {
do_compress_device_host(params);
}
private:
CUTE_HOST_DEVICE
static void
do_compress_device_host(Params params) {
auto [m, n, k, l] = params.problem_shape;
auto [ptr_A, dA, ptr_ACompress, ptr_E] = params.transform;
auto workspace = params.workspace;
const int aligned_k = (k + TensorAAlignmentK - 1) / TensorAAlignmentK * TensorAAlignmentK;
const int aligned_m = (m + TensorAAlignmentM - 1) / TensorAAlignmentM * TensorAAlignmentM;
const int metadata_k = (k + TensorEAlignmentK - 1) / TensorEAlignmentK * TensorEAlignmentK;
const int metadata_m = (m + TensorEAlignmentM - 1) / TensorEAlignmentM * TensorEAlignmentM;
const int k_compressed = aligned_k / ElementASparsity{};
// Convert to CuTe tensors. But don't want to use sparse_ptr, which is making everything complicated here.
cute::Tensor tensorA = make_tensor(recast_ptr<ElementAUint>(ptr_A), make_layout(make_shape(m, k, l), dA));
cute::Tensor tensorAc = make_tensor(recast_ptr<ElementAUint>(ptr_ACompress),
make_shape(aligned_m, k_compressed, l),
make_cute_packed_stride(StrideA{}, cute::make_shape(aligned_m, k_compressed, l)));
cute::Tensor tensorE_raw_compress_logical = make_tensor(recast_ptr<sparse_elem<ElementEMmaSparsity{},ElementEMmaRaw>>(workspace),
make_shape(metadata_m, make_shape(TensorEAtomK{}, metadata_k / TensorEAtomK{}), l),
make_stride(TensorEAtomK{}, make_stride(_1{}, metadata_m*TensorEAtomK{}), metadata_m*metadata_k));
cute::Tensor tensorE_raw_compress = recast<uint8_t>(tensorE_raw_compress_logical);
// The following vars are all logical.
int atom_m = size<0>(TensorEAtom{});
int atom_k = size<1>(TensorEAtom{});
int tiled_m = metadata_m / atom_m;
int tiled_ke = metadata_k / atom_k;
// Col major when viewing atoms
int stride_tile_m = cosize(TensorEAtom{});
int stride_tile_ke = atom_k * metadata_m;
// Logical metadata tensor
cute::Tensor tensorE_logical = make_tensor(recast_ptr<sparse_elem<ElementEMmaSparsity{},ElementEMmaRaw>>(ptr_E),
make_layout(make_shape(append(shape<0>(TensorEAtom{}), tiled_m),
append(shape<1>(TensorEAtom{}), tiled_ke),
shape<2>(tensorE_raw_compress_logical)),
make_stride(append(stride<0>(TensorEAtom{}), stride_tile_m),
append(stride<1>(TensorEAtom{}), stride_tile_ke),
stride<2>(tensorE_raw_compress_logical))));
// Physical metadata tensor
cute::Tensor tensorE = recast<uint8_t>(tensorE_logical);
// void do_init()
cute::clear(tensorAc);
cute::clear(tensorE_raw_compress);
// void do_raw_compress()
using TileStepA = Int<LogicalElemsAPerChunk * 2>;
using TileStepAc = Int<TileStepA{} / 2>;
cute::Tensor tensorATiled = logical_divide(tensorA, make_shape(_, TileStepA{}, _));
cute::Tensor tensorAcTiled = logical_divide(tensorAc, make_shape(_, TileStepAc{}, _));
for (int batch_idx = 0; batch_idx < l; batch_idx++) {
for (int m_idx = 0; m_idx < m; m_idx++) {
for (int tiler_k_idx = 0; tiler_k_idx < size<1,1>(tensorATiled); tiler_k_idx++) {
int effective_elems = cute::min(TileStepA{}, k - (tiler_k_idx * TileStepA{}));
detail::compress_two_chunks_legacy<SparseConfig>(tensorATiled(m_idx, make_coord(_, tiler_k_idx), batch_idx),
tensorAcTiled(m_idx, make_coord(_, tiler_k_idx), batch_idx),
tensorE_raw_compress(m_idx, tiler_k_idx, batch_idx),
effective_elems);
}
}
}
// void do_reorder()
// Fast path when we don't permute.
if constexpr (sizeof_bits_v<ElementAUint> <= 8) {
memcpy(tensorE.data(), tensorE_raw_compress.data(), tensorE.size());
}
else {
cute::copy(tensorE_raw_compress, tensorE);
}
#if 0
print("--> TensorA\n");
auto tensorA_eltA = cute::recast<ElementA>(tensorA);
cute::print_tensor(tensorA_eltA); printf("\n\n");
print("--> REF TensorAC\n");
auto tensorAc_eltA = cute::recast<ElementA>(tensorAc);
cute::print_tensor(tensorAc_eltA); printf("\n\n");
print("--> REF TensorE\n");
cute::print_tensor(tensorE); printf("\n\n");
#endif
}
};
} // namespace kernel
} // namespace transform
} // namespace cutlass
test/unit/transform/device/testbed_sparse_gemm_compressor.hpp
+876
View File
@@ -0,0 +1,876 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*
* @brief Test for structured sparse gemm compressor device kernel
*/
#pragma once
#include <cuda_runtime_api.h> // cudaGetLastError
#include <cstdint> // uint64_t
#include <cstdio> // printf
#include <cstdlib> // malloc
#include <iostream> // std::cout
#include <vector>
#include <array>
#include "cute/layout.hpp" // cute::make_shape
#include "cute/util/type_traits.hpp" // cute::is_same_v
#include "cutlass/coord.h" // cutlass::make_Coord
#include "cutlass/cutlass.h" // cutlass::Status
#include "cutlass/kernel_hardware_info.hpp" // cutlass::KernelHardwareInfo
#include "cutlass/layout/matrix.h" // cutlass::layout::Affine2Layout_Factory
#include "cutlass/numeric_types.h" // cutlass::sizeof_bits, cutlass::float_
#include "cutlass/tensor_view.h" // cutlass::TensorView
#include "cutlass/transform/device/transform_universal_adapter.hpp" // cutlass::transform::device::TransformUniversalAdapter
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp" // cutlass::transform::kernel::StructuredSparseCompressorUtility
#include "cutlass/util/device_memory.h" // cutlass::device_memory::allocation
#include "cutlass/util/distribution.h" // cutlass::Distribution
#include "cutlass/util/host_tensor.h" // cutlass::HostTensor
#include "cutlass/util/packed_stride.hpp" // cutlass::make_cute_packed_stride
#include "cutlass/util/reference/host/tensor_compare.h" // cutlass::reference::host::TensorEquals
#include "cutlass/util/reference/host/tensor_fill.h" // cutlass::reference::host::TensorFillRandomUniform, TensorFillIdentity, TensorFillRandomGaussian, BlockFillSequential, TensorFill
#include "sm90_sparse_gemm_compressor_legacy.hpp" // Legacy host compressor
#include "../../common/cutlass_unit_test.h" // CUTLASS UT, EXPECT_TRUE
#define CUDA_CHECK_FALSE(cuda_error) \
{ \
if (cuda_error != cudaSuccess) { \
printf("cudaError %s in %s:%d\n", cudaGetErrorString(cuda_error), __func__, __LINE__ ); \
return false; \
} \
}
#define CUDA_CHECK(cuda_error) \
{ \
if (cuda_error != cudaSuccess) { \
printf("cudaError %s in %s:%d\n", cudaGetErrorString(cuda_error), __func__, __LINE__ ); \
return; \
} \
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// * Test Bed
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace test
{
namespace transform
{
namespace device
{
// Helper Functions
template <typename Element, typename Layout>
bool
initialize_tensor(cutlass::TensorView<Element, Layout> view, cutlass::Distribution::Kind dist_kind, uint64_t seed)
{
if (dist_kind == cutlass::Distribution::Uniform) {
double scope_max, scope_min;
int bits_input = cutlass::sizeof_bits<Element>::value;
if (bits_input == 1) {
scope_max = 2;
scope_min = 0;
}
else if (bits_input <= 8) {
scope_max = 1;
scope_min = -1;
} else {
scope_max = 4;
scope_min = -4;
}
cutlass::reference::host::TensorFillRandomUniform(view, seed, scope_max, scope_min, 0);
}
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
}
else if (dist_kind == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(view.data(), view.capacity());
}
else if (dist_kind == cutlass::Distribution::AllOnes) {
cutlass::reference::host::TensorFill(view, Element(1));
}
else if (dist_kind == cutlass::Distribution::AllZeros) {
cutlass::reference::host::TensorFill(view, Element(0));
}
else {
EXPECT_TRUE(false) << "Not implemented";
return false;
}
return true;
}
// Testbed
template <typename Compressor_>
struct TestbedSparseGemmCompressor {
public:
using Compressor = Compressor_;
using CompressorKernel = typename Compressor::TransformKernel;
using ElementA = typename CompressorKernel::ElementA;
using LayoutATag = typename CompressorKernel::LayoutATag;
using StrideA = typename CompressorKernel::StrideA;
using ArrayElementA =
ElementA
;
using ElementE = typename CompressorKernel::ElementEMmaRaw;
using LayoutETag = cutlass::layout::RowMajor; // We don't care about the major here, just to allocate tensor
using SparseConfig = typename CompressorKernel::SparseConfig;
using ProblemShapeType = typename CompressorKernel::ProblemShape;
using CompressorUtility = cutlass::transform::kernel::StructuredSparseCompressorUtility<
ProblemShapeType,
ElementA,
LayoutATag,
SparseConfig>;
using CompressorKernelHost = cutlass::transform::kernel::SM90StructuredSparseCompressorLegacy<
ProblemShapeType,
ElementA,
LayoutATag,
SparseConfig>;
using CompressorHost = cutlass::transform::device::TransformUniversalAdapter<CompressorKernelHost>;
static constexpr auto LogicalElemsAPerChunk = CompressorKernel::LogicalElemsAPerChunk;
static constexpr auto PhysicalElemsAPerChunk = CompressorKernel::PhysicalElemsAPerChunk;
struct Data {
// Data Storage
cutlass::HostTensor<ArrayElementA, LayoutATag> tensor_A;
cutlass::HostTensor<ArrayElementA, LayoutATag> tensor_A_Comp;
cutlass::HostTensor<ElementE, LayoutETag> tensor_E;
cutlass::HostTensor<ArrayElementA, LayoutATag> tensor_A_Comp_ref;
cutlass::HostTensor<ElementE, LayoutETag> tensor_E_ref;
};
struct CudaRAII {
cudaStream_t stream;
cudaEvent_t start;
cudaEvent_t stop;
CudaRAII(){
CUDA_CHECK(cudaStreamCreate( &stream ));
CUDA_CHECK(cudaEventCreate( &start ));
CUDA_CHECK(cudaEventCreate( &stop ));
};
CudaRAII(const CudaRAII&) = delete;
CudaRAII& operator=(const CudaRAII&) = delete;
CudaRAII(CudaRAII&&) = delete;
CudaRAII& operator=(CudaRAII&&) = delete;
~CudaRAII(){
CUDA_CHECK(cudaStreamDestroy( stream ));
CUDA_CHECK(cudaEventDestroy( start ));
CUDA_CHECK(cudaEventDestroy( stop ));
}
};
public:
TestbedSparseGemmCompressor(
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_E_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_A_Comp_ = cutlass::Distribution::Uniform,
uint64_t seed_ = 7)
: init_A(init_A_)
, init_E(init_E_)
, init_A_Comp(init_A_Comp_)
, seed(seed_)
{
}
bool valid_test(ProblemShapeType problem_shape_MNKL)
{
const int GemmK = cute::size<2>(problem_shape_MNKL);
if ( GemmK % LogicalElemsAPerChunk != 0 ) {
printf("GemmK needs to be multiplier of LogicalElemsAPerChunk\n");
return false;
}
return true;
}
bool initialize(ProblemShapeType problem_shape_MNKL, Data& datas)
{
CUDA_CHECK_FALSE(cudaGetLastError());
// In unit of ElementARaw
const int GemmM = cute::size<0>(problem_shape_MNKL);
const int GemmN = cute::size<1>(problem_shape_MNKL);
const int GemmK = cute::size<2>(problem_shape_MNKL);
const int GemmL = cute::size<3>(problem_shape_MNKL);
// Compressor utility to get allocated data size
auto stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(GemmM, GemmK, GemmL));
CompressorUtility compressor_utility(problem_shape_MNKL, stride_a);
// TensorA
// In unit of ElementARaw, after alignment requirement
// M-dim: no alignment requirement
// K-dim: multiplier of chunk size
// TensorA Compressed
// In unit of ElementARaw, after alignment requirement
// M-dim: TMA alignment
// K-dim: TMA alignment
const int GemmMAlignedAC = compressor_utility.get_tensorA_m_physical();
const int GemmKAlignedAC = compressor_utility.get_tensorA_k_physical();
// TensorE
// In unit of ElementE (uint8_t), after alignment requirement
// M-dim: TensorEAtom_M alignment
// K-dim: TensorEAtom_K alignment
const int GemmMAlignedE = compressor_utility.get_metadata_m_physical();
const int GemmKAlignedE = compressor_utility.get_metadata_k_physical();
auto a_coord = cutlass::make_Coord(GemmM * GemmL, GemmK);
auto e_coord = cutlass::make_Coord(GemmMAlignedE * GemmL, GemmKAlignedE);
auto a_comp_coord = cutlass::make_Coord(GemmMAlignedAC * GemmL, GemmKAlignedAC);
typename LayoutATag::Stride stride_factor_A;
typename LayoutETag::Stride stride_factor_E;
datas.tensor_A.resize(a_coord,
cutlass::layout::Affine2Layout_Factory<LayoutATag>::layout_factory(a_coord, stride_factor_A));
datas.tensor_A_Comp.resize(a_comp_coord,
cutlass::layout::Affine2Layout_Factory<LayoutATag>::layout_factory(a_comp_coord, stride_factor_A));
datas.tensor_A_Comp_ref.resize(a_comp_coord,
cutlass::layout::Affine2Layout_Factory<LayoutATag>::layout_factory(a_comp_coord, stride_factor_A),
false);
datas.tensor_E.resize(e_coord,
cutlass::layout::Affine2Layout_Factory<LayoutETag>::layout_factory(e_coord, stride_factor_E));
datas.tensor_E_ref.resize(e_coord,
cutlass::layout::Affine2Layout_Factory<LayoutETag>::layout_factory(e_coord, stride_factor_E),
false);
EXPECT_TRUE(initialize_tensor(datas.tensor_A.host_view(), init_A, seed + 1));
EXPECT_TRUE(initialize_tensor(datas.tensor_E.host_view(), init_E, seed + 2));
EXPECT_TRUE(initialize_tensor(datas.tensor_E_ref.host_view(), init_E, seed + 3));
EXPECT_TRUE(initialize_tensor(datas.tensor_A_Comp.host_view(), init_A_Comp, seed + 4));
EXPECT_TRUE(initialize_tensor(datas.tensor_A_Comp_ref.host_view(), init_A_Comp, seed + 5));
compressor_utility.structure_sparse_zero_mask_fill(datas.tensor_A.host_data(), seed + 6);
// Check for failed devide
CUDA_CHECK_FALSE(cudaGetLastError());
datas.tensor_A.sync_device();
datas.tensor_A_Comp.sync_device();
datas.tensor_E.sync_device();
// Check for failed devide
CUDA_CHECK_FALSE(cudaGetLastError());
return true;
}
bool run_device(ProblemShapeType problem_shape_MNKL, Data& datas, float* time = nullptr)
{
CudaRAII cuda_raii;
const int GemmM = cute::size<0>(problem_shape_MNKL);
const int GemmN = cute::size<1>(problem_shape_MNKL);
const int GemmK = cute::size<2>(problem_shape_MNKL);
const int GemmL = cute::size<3>(problem_shape_MNKL);
StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(GemmM, GemmK, GemmL));
cutlass::KernelHardwareInfo hw_info;
hw_info.device_id = 0;
hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);
typename Compressor::Arguments arguments{
{GemmM, GemmN, GemmK, GemmL},
{datas.tensor_A.device_data(),
stride_a,
datas.tensor_A_Comp.device_data(),
datas.tensor_E.device_data()},
{hw_info}
};
Compressor compressor_op;
size_t workspace_size = Compressor::get_workspace_size(arguments);
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
cutlass::Status status {cutlass::Status::kSuccess };
status = compressor_op.can_implement(arguments);
if (status != cutlass::Status::kSuccess) {
CUDA_CHECK_FALSE(cudaGetLastError());
}
status = compressor_op.initialize(arguments, workspace.get(), cuda_raii.stream);
if (status != cutlass::Status::kSuccess) {
CUDA_CHECK_FALSE(cudaGetLastError());
}
CUDA_CHECK_FALSE(cudaStreamSynchronize(cuda_raii.stream));
CUDA_CHECK_FALSE(cudaEventRecord(cuda_raii.start, cuda_raii.stream));
status = compressor_op.run(cuda_raii.stream);
if (status != cutlass::Status::kSuccess) {
CUDA_CHECK_FALSE(cudaGetLastError());
}
CUDA_CHECK_FALSE(cudaEventRecord(cuda_raii.stop, cuda_raii.stream));
CUDA_CHECK_FALSE(cudaEventSynchronize(cuda_raii.stop));
CUDA_CHECK_FALSE(cudaStreamSynchronize(cuda_raii.stream));
if ( time != nullptr ){
CUDA_CHECK_FALSE(cudaEventElapsedTime(time, cuda_raii.start, cuda_raii.stop));
}
datas.tensor_A_Comp.sync_host();
datas.tensor_E.sync_host();
#if 0
{
printf("\n--> DEVICE OUTPUT\n");
printf("datas.tensor_A\n");
std::cout << datas.tensor_A.host_view() << std::endl << std::endl;
printf("datas.tensor_A_Comp\n");
std::cout << datas.tensor_A_Comp.host_view() << std::endl << std::endl;
printf("datas.tensor_E\n");
std::cout << datas.tensor_E.host_view() << std::endl << std::endl;
}
#endif
return true;
}
bool run_host_ref(ProblemShapeType problem_shape_MNKL, Data& datas)
{
const int GemmM = cute::size<0>(problem_shape_MNKL);
const int GemmN = cute::size<1>(problem_shape_MNKL);
const int GemmK = cute::size<2>(problem_shape_MNKL);
const int GemmL = cute::size<3>(problem_shape_MNKL);
StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(GemmM, GemmK, GemmL));
typename CompressorKernelHost::Arguments arguments{
{GemmM, GemmN, GemmK, GemmL},
{datas.tensor_A.host_data(),
stride_a,
datas.tensor_A_Comp_ref.host_data(),
datas.tensor_E_ref.host_data()},
{}};
const auto can_imp = CompressorKernelHost::can_implement(arguments);
if (can_imp != cutlass::Status::kSuccess) {
printf("can_implement() check failed\n");
return false;
}
// Relies on std::vector for RAII
auto workspace_size =
static_cast<std::vector<uint8_t>::size_type>(CompressorKernelHost::get_workspace_size(arguments));
std::vector<uint8_t> workspace_vector(workspace_size);
auto workspace = static_cast<void*>(workspace_vector.data());
cutlass::Status status = CompressorKernelHost::initialize_workspace(arguments, workspace);
if (status != cutlass::Status::kSuccess) {
printf("initialize_workspace() failed\n");
return false;
}
auto params = CompressorKernelHost::to_underlying_arguments(arguments, workspace);
CompressorKernelHost::run(params);
return true;
}
bool compare_reference(Data& datas)
{
bool check_tensor_a_compressed =
cutlass::reference::host::TensorEquals(datas.tensor_A_Comp_ref.host_view(), datas.tensor_A_Comp.host_view());
if (!check_tensor_a_compressed) {
printf("A-Compressed Mismatch\n");
}
bool check_tensor_e = cutlass::reference::host::TensorEquals(datas.tensor_E_ref.host_view(), datas.tensor_E.host_view());
if (!check_tensor_e) {
printf("E Mismatch\n");
}
return check_tensor_a_compressed && check_tensor_e;
}
bool run_auto_small()
{
return run_auto(true);
}
bool run_auto(bool run_small = false)
{
constexpr auto TensorEAlignmentM = typename SparseConfig::TensorEAlignmentM{};
constexpr auto TensorEAlignmentK = typename SparseConfig::TensorEAlignmentK{};
constexpr int LogicalElemsAPerChunk = typename SparseConfig::LogicalElemsAPerChunk{};
constexpr int GemmN = 1;
using ProblemType = typename std::array<int, 4>;
std::vector<ProblemType> problems;
const std::vector<ProblemType> problems_multiplier_of_tensor_e_atom = {
// * Regular Cases (multiplier of TensorEAlignment)
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 2, 1},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 2, 1},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 3, 1},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 2, 1},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 2, 1},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 3, 1},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 2, 1},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 2, 1},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 3, 1},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 2, 2},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 2, 2},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 3, 2},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 2, 2},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 2, 2},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 3, 2},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 2, 2},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 2, 2},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 3, 2},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 2, 3},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 2, 3},
{TensorEAlignmentM * 1, GemmN, TensorEAlignmentK * 3, 3},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 2, 3},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 2, 3},
{TensorEAlignmentM * 2, GemmN, TensorEAlignmentK * 3, 3},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 2, 3},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 2, 3},
{TensorEAlignmentM * 3, GemmN, TensorEAlignmentK * 3, 3},
};
const std::vector<ProblemType> problems_multiplier_of_tensor_e_atom_large = {
// * Large Case (multiplier of TensorEAlignment)
{TensorEAlignmentM * 10, GemmN, TensorEAlignmentK * 13, 1},
// {TensorEAlignmentM * 11, GemmN, TensorEAlignmentK * 14, 2},
// {TensorEAlignmentM * 12, GemmN, TensorEAlignmentK * 15, 3},
};
const std::vector<ProblemType> problems_multiplier_of_twochunk {
// * Corner Cases
{4, GemmN, LogicalElemsAPerChunk * 2, 1},
{4, GemmN, LogicalElemsAPerChunk * 4, 1},
{4, GemmN, LogicalElemsAPerChunk * 6, 1},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 1},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 1},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 1},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 1},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 1},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 1},
{4, GemmN, LogicalElemsAPerChunk * 2, 2},
{4, GemmN, LogicalElemsAPerChunk * 4, 2},
{4, GemmN, LogicalElemsAPerChunk * 6, 2},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 2},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 2},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 2},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 2},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 2},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 2},
{4, GemmN, LogicalElemsAPerChunk * 2, 3},
{4, GemmN, LogicalElemsAPerChunk * 4, 3},
{4, GemmN, LogicalElemsAPerChunk * 6, 3},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 3},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 3},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 3},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 3},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 3},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 3},
{32 + 4, GemmN, LogicalElemsAPerChunk * 2, 1},
{32 + 4, GemmN, LogicalElemsAPerChunk * 4, 1},
{32 + 4, GemmN, LogicalElemsAPerChunk * 6, 1},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 1},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 1},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 1},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 1},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 1},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 1},
{32 + 4, GemmN, LogicalElemsAPerChunk * 2, 2},
{32 + 4, GemmN, LogicalElemsAPerChunk * 4, 2},
{32 + 4, GemmN, LogicalElemsAPerChunk * 6, 2},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 2},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 2},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 2},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 2},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 2},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 2},
{32 + 4, GemmN, LogicalElemsAPerChunk * 2, 3},
{32 + 4, GemmN, LogicalElemsAPerChunk * 4, 3},
{32 + 4, GemmN, LogicalElemsAPerChunk * 6, 3},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 3},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 3},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 3},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 3},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 3},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 3},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 2, 1},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 4, 1},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 6, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 1},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 2, 2},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 4, 2},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 6, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 2},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 2, 3},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 4, 3},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 6, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 3},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 2, 1},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 4, 1},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 6, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 1},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 2, 2},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 4, 2},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 6, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 2},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 2, 3},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 4, 3},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 6, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 2, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 4, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 6, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 2, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 4, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 6, 3},
};
const std::vector<ProblemType> problems_multiplier_of_onechunk {
{4, GemmN, LogicalElemsAPerChunk * 1, 1},
{4, GemmN, LogicalElemsAPerChunk * 3, 1},
{4, GemmN, LogicalElemsAPerChunk * 5, 1},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 1},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 1},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 1},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 1},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 1},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 1},
{4, GemmN, LogicalElemsAPerChunk * 1, 2},
{4, GemmN, LogicalElemsAPerChunk * 3, 2},
{4, GemmN, LogicalElemsAPerChunk * 5, 2},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 2},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 2},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 2},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 2},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 2},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 2},
{4, GemmN, LogicalElemsAPerChunk * 1, 3},
{4, GemmN, LogicalElemsAPerChunk * 3, 3},
{4, GemmN, LogicalElemsAPerChunk * 5, 3},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 3},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 3},
{4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 3},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 3},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 3},
{4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 3},
{32 + 4, GemmN, LogicalElemsAPerChunk * 1, 1},
{32 + 4, GemmN, LogicalElemsAPerChunk * 3, 1},
{32 + 4, GemmN, LogicalElemsAPerChunk * 5, 1},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 1},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 1},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 1},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 1},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 1},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 1},
{32 + 4, GemmN, LogicalElemsAPerChunk * 1, 2},
{32 + 4, GemmN, LogicalElemsAPerChunk * 3, 2},
{32 + 4, GemmN, LogicalElemsAPerChunk * 5, 2},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 2},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 2},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 2},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 2},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 2},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 2},
{32 + 4, GemmN, LogicalElemsAPerChunk * 1, 3},
{32 + 4, GemmN, LogicalElemsAPerChunk * 3, 3},
{32 + 4, GemmN, LogicalElemsAPerChunk * 5, 3},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 3},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 3},
{32 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 3},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 3},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 3},
{32 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 3},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 1, 1},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 3, 1},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 5, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 1},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 1},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 1, 2},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 3, 2},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 5, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 2},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 2},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 1, 3},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 3, 3},
{TensorEAlignmentM + 4, GemmN, LogicalElemsAPerChunk * 5, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 3},
{TensorEAlignmentM + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 3},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 1, 1},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 3, 1},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 5, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 1},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 1},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 1, 2},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 3, 2},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 5, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 2},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 2},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 1, 3},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 3, 3},
{TensorEAlignmentM * 2 + 4, GemmN, LogicalElemsAPerChunk * 5, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 1, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 3, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK + LogicalElemsAPerChunk * 5, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 1, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 3, 3},
{TensorEAlignmentM * 2 + 4, GemmN, TensorEAlignmentK * 2 + LogicalElemsAPerChunk * 5, 3},
};
// Run small only run multiplier of chunk size cases
if (run_small) {
problems.insert(problems.end(), problems_multiplier_of_tensor_e_atom.begin(), problems_multiplier_of_tensor_e_atom.end());
}
// Run full run all corner cases
else {
problems.insert(problems.end(), problems_multiplier_of_tensor_e_atom_large.begin(), problems_multiplier_of_tensor_e_atom_large.end());
problems.insert(problems.end(), problems_multiplier_of_tensor_e_atom.begin(), problems_multiplier_of_tensor_e_atom.end());
problems.insert(problems.end(), problems_multiplier_of_twochunk.begin(), problems_multiplier_of_twochunk.end());
problems.insert(problems.end(), problems_multiplier_of_onechunk.begin(), problems_multiplier_of_onechunk.end());
}
for (const auto& problem_shape_MNKL : problems) {
const auto [GemmM, GemmN, GemmK, GemmL] = problem_shape_MNKL;
bool passed = run({GemmM, GemmN, GemmK, GemmL});
printf("run() (%.4d,%.4d,%.4d,%.4d) %s\n", GemmM, GemmN, GemmK, GemmL, passed ? "PASS" : "FAIL");
CUTLASS_TRACE_HOST("run() " << GemmM << " " << GemmN << " " << GemmK << " " << GemmL << passed ? " PASS" : " FAIL");
if (not passed) {
return false;
}
}
return true;
}
bool run(ProblemShapeType problem_shape_MNKL)
{
// Check if valid test
if (not valid_test(problem_shape_MNKL)) {
CUTLASS_TRACE_HOST("valid_test() fail\n");
return false;
}
// Data Storage
Data datas;
// Initialize Data
if (not initialize(problem_shape_MNKL, datas)) {
CUTLASS_TRACE_HOST("initialize() fail\n");
return false;
}
// Run Compressor (Host Ref)
if (not run_host_ref(problem_shape_MNKL, datas)) {
CUTLASS_TRACE_HOST("run_host() fail\n");
return false;
}
// Run Compressor (Device)
if (not run_device(problem_shape_MNKL, datas)) {
CUTLASS_TRACE_HOST("run_device() fail\n");
return false;
}
// Verify
if (not compare_reference(datas)) {
CUTLASS_TRACE_HOST("compare_reference() DEVICE <-> LEGACY HOST fail\n");
printf("compare_reference() DEVICE <-> LEGACY HOST fail\n");
return false;
}
// else {
// printf("DEVICE <-> HOST PASS\n");
// }
return true;
}
bool benchmark(ProblemShapeType problem_shape_MNKL) {
const auto [GemmM, GemmN, GemmK, GemmL] = problem_shape_MNKL;
printf("Benchmark() (%.4d,%.4d,%.4d,%.4d) START\n", GemmM, GemmN, GemmK, GemmL);
// Check if valid test
if (valid_test(problem_shape_MNKL) == false) {
CUTLASS_TRACE_HOST("valid_test() fail\n");
return false;
}
// 2 warm-up iterations and 10 timing iterations
constexpr int num_warmup = 5;
constexpr int num_iter = 10;
// Duplicate data to mimic cold cache
Data data[num_warmup + num_iter];
double total_time_milliseconds{0.0};
for (int i = 0; i < num_warmup + num_iter; ++i ) {
printf("Benchmark() (%.4d,%.4d,%.4d,%.4d) ITER %d\n", GemmM, GemmN, GemmK, GemmL, i );
auto& datum_i = data[i];
// Initialize Data
if (initialize(problem_shape_MNKL, datum_i) == false) {
CUTLASS_TRACE_HOST("initialize() fail\n");
return false;
}
// Run Compressor (Device)
double time_i_milliseconds{0.0f};
if (not run_device(problem_shape_MNKL, datum_i, &time_i_milliseconds)) {
CUTLASS_TRACE_HOST("run_device() fail\n");
return false;
}
if ( i >= num_warmup ) {
total_time_milliseconds += time_i_milliseconds;
}
}
const double mean_time_milliseconds = total_time_milliseconds / num_iter;
printf("Mean time (ms): %.5f\n", mean_time_milliseconds);
return true;
}
public:
// Data Init Setting
cutlass::Distribution::Kind init_A;
cutlass::Distribution::Kind init_A_Comp;
cutlass::Distribution::Kind init_E;
uint64_t seed;
};
} // namespace device
} // namespace transform
} // namespace test