leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

CUTLASS 3.5.1 (#1623)

Browse Source 
* CUTLASS 3.5.1

* updates, optimizations, fixes
This commit is contained in:
Vijay Thakkar
2024-07-29 08:46:24 -04:00
committed by GitHub
parent 56b46e2d13
commit be60a0b272
312 changed files with 19793 additions and 6775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/CMakeLists.txt
+4
View File
@@ -33,3 +33,7 @@ else()
add_custom_target(test_unit)
endif()
if (CUTLASS_ENABLE_SELF_CONTAINED_INCLUDES_CHECK)
add_subdirectory(self_contained_includes)
endif()
test/self_contained_includes/CMakeLists.txt
+57
View File
@@ -0,0 +1,57 @@
# Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# The purpose of this target is to check if the following header files are self-contained,
# i.e. they can be included in a source file without needing to include other headers before it.
set(header_files_to_check
cutlass/gemm/kernel/default_gemm.h
cutlass/gemm/kernel/default_gemm_complex.h
cutlass/gemm/kernel/gemm_universal_decl.h
# cutlass/gemm/kernel/sm90_gemm_warpspecialized.hpp
cute/tensor_impl.hpp
)
# for each header in _header_files:
# create a .cu file with the same name as the header's path, except with / replaced with %
# have the .cu file include that header
set(_gen_source_files "")
foreach(header_file ${header_files_to_check})
string(REPLACE "/" "%" header_file_esc ${header_file})
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${header_file_esc}.cu"
"#include <${header_file}>")
list(APPEND _gen_source_files
"${CMAKE_CURRENT_BINARY_DIR}/${header_file_esc}.cu")
endforeach()
# build all generated .cu files into a single library
cutlass_add_library(test_self_contained_includes MODULE ${_gen_source_files})
test/unit/conv/device/conv2d_fprop_implicit_gemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32_sm89.cu
+7 -7
View File
@@ -60,7 +60,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc_
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -104,7 +104,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe5m2nhwc_fe4m3nhwc_fe4m3nhw
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -148,7 +148,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe5m2nhwc_fe4m3nhwc_fe5m2nhw
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e5m2_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -192,7 +192,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -236,7 +236,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -280,7 +280,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -324,7 +324,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
test/unit/conv/device/conv2d_fprop_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Conv2dFpropSM80TestResidaulBlock() {
static void Conv2dFpropSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Conv2dFpropSM80TestResidaulBlock() {
TEST(SM80_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Conv2dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_fprop_with_broadcast_sm70.cu
+2 -2
View File
@@ -60,7 +60,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = false
>
void TestResidaulBlock() {
void Conv2dFpropSM70TestResidualBlock() {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
@@ -117,7 +117,7 @@ void TestResidaulBlock() {
TEST(SM70_Device_Conv2d_Fprop_With_Residual_Block_Plus_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// Resnet
TestResidaulBlock<float, cutlass::epilogue::thread::ReLu, cutlass::plus, cutlass::epilogue::thread::Identity>();
Conv2dFpropSM70TestResidualBlock<float, cutlass::epilogue::thread::ReLu, cutlass::plus, cutlass::epilogue::thread::Identity>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_fprop_with_broadcast_sm75.cu
+3 -3
View File
@@ -103,7 +103,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
void Conv2dFpropSM75TestResidualBlock() {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
@@ -160,14 +160,14 @@ void TestResidaulBlock() {
TEST(SM75_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// Resnet
TestResidaulBlock<cutlass::half_t, cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM75TestResidualBlock<cutlass::half_t, cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
TEST(SM75_Device_Conv2d_Fprop_With_Residual_Block_Multiply_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// EfficientNet V2
// Do not run split-K tests since the activation op is not Identity.
TestResidaulBlock<float, cutlass::epilogue::thread::Sigmoid, cutlass::multiplies, cutlass::epilogue::thread::Identity, false>();
Conv2dFpropSM75TestResidualBlock<float, cutlass::epilogue::thread::Sigmoid, cutlass::multiplies, cutlass::epilogue::thread::Identity, false>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_fprop_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Conv3dFpropSM80TestResidaulBlock() {
static void Conv3dFpropSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Conv3dFpropSM80TestResidaulBlock() {
TEST(SM80_Device_Conv3d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Conv3dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv3dFpropSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_with_broadcast_testbed.h
+37 -20
View File
@@ -204,17 +204,29 @@ public:
}
void initialize(
cutlass::conv::Conv3dProblemSize const &problem_size, uint64_t seed = 2019) {
cutlass::conv::Conv3dProblemSize const &problem_size, bool non_packed_test = false, uint64_t seed = 2019) {
tensor_A.resize(implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size));
tensor_B.resize(implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size));
tensor_C.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_C_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Z_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Z_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
// to make the layout of tensors a little bit bigger than the problem size
cutlass::Tensor5DCoord stride_increment = cutlass::Tensor5DCoord(8, 16, 32, 32, 64);
cutlass::Tensor5DCoord tensor_A_extent = implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size);
cutlass::Tensor5DCoord tensor_B_extent = implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size);
cutlass::Tensor5DCoord tensor_C_extent = implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size);
if (non_packed_test) {
tensor_A_extent += stride_increment;
tensor_C_extent += stride_increment;
}
tensor_A.resize(tensor_A_extent);
tensor_B.resize(tensor_B_extent);
tensor_C.resize(tensor_C_extent);
tensor_C_reference.resize(tensor_C_extent);
tensor_Z_computed.resize(tensor_C_extent);
tensor_Z_reference.resize(tensor_C_extent);
tensor_T_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_T_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Y_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Y_reference.resize(tensor_C_extent);
tensor_Broadcast.resize({
1,
1,
@@ -282,6 +294,7 @@ public:
bool run(
cutlass::conv::Conv3dProblemSize const &problem_size,
cutlass::conv::SplitKMode const &split_k_mode = cutlass::conv::SplitKMode::kSerial,
bool non_packed_test = false,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(1)) {
@@ -300,7 +313,7 @@ public:
<< std::endl;
#endif
initialize(problem_size);
initialize(problem_size, non_packed_test);
// configure the operator
Conv3d conv3d_op;
@@ -479,6 +492,7 @@ public:
<< problem_size.dilation_h << "x"
<< problem_size.dilation_w << "_"
<< (problem_size.mode == cutlass::conv::Mode::kCrossCorrelation ? "xcorr_" : "conv_")
<< (non_packed_test ? "non_packed_tensor_test_" : "packed_tensor_test_")
<< Conv3d::ThreadblockShape::kM << "x"
<< Conv3d::ThreadblockShape::kN << "x"
<< Conv3d::ThreadblockShape::kK << "_"
@@ -521,7 +535,8 @@ template <typename ImplicitGemm,
>
bool TestAllConv3dWithBroadcast(
const Conv3dProblemVector &conv_test_sizes = Conv3dProblemVector(),
const Conv3dProblemVector &conv_blacklist_sizes = Conv3dProblemVector()) {
const Conv3dProblemVector &conv_blacklist_sizes = Conv3dProblemVector(),
bool non_packed_test = false) {
bool passed = true;
@@ -595,17 +610,17 @@ bool TestAllConv3dWithBroadcast(
// test mode = xcross
passed = testbed.run(
conv_problem,
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
// test mode = convolution
passed = testbed.run(
conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
@@ -651,6 +666,7 @@ bool TestAllConv3dWithBroadcast(
passed = testbed.run(
conv3d_split_k_test_size.reset_split_k_slices(split_k_slice),
split_k_mode,
false,/*non_packed_test*/
cutlass::from_real<typename ImplicitGemm::ElementCompute>(alpha),
cutlass::from_real<typename ImplicitGemm::ElementCompute>(beta));
@@ -669,7 +685,8 @@ template <typename ImplicitGemm,
typename ReferenceOp = Conv3dWithBroadcastReferenceOp<ImplicitGemm>,
bool AddBroadcastFirst = false>
bool TestSpecificConv3dWithBroadcast(
const Conv3dProblemVector & problem_sizes) {
const Conv3dProblemVector & problem_sizes,
bool non_packed_test = false) {
bool passed = true;
@@ -686,19 +703,19 @@ bool TestSpecificConv3dWithBroadcast(
// Test
//
// test mode = xcross
// test mode = xcross, non_packed_test = false
passed = testbed.run(
conv_problem,
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
// test mode = convolution
// test mode = convolution, non_packed_test = false
passed = testbed.run(
conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
test/unit/conv/device/deconv2d_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv2dSM80TestResidaulBlock() {
static void Deconv2dSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -163,7 +163,7 @@ static void Deconv2dSM80TestResidaulBlock() {
TEST(SM80_Device_Deconv2d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv2dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Deconv2dSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/deconv3d_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -103,7 +103,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv3dSM80TestResidaulBlock() {
static void Deconv3dSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Deconv3dSM80TestResidaulBlock() {
TEST(SM80_Device_Deconv3d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv3dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Deconv3dSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device_3x/conv_problem_sizes.hpp
+181 -9
View File
@@ -42,7 +42,7 @@ namespace test::conv::device {
/////////////////////////////////////////////////////////////////////////////////////////////////
template<int SpatialDim, cutlass::conv::Operator ConvOp>
template<int SpatialDim, cutlass::conv::Operator ConvOp, bool SupportStrides = (ConvOp != cutlass::conv::Operator::kDgrad)>
std::vector<cutlass::conv::ConvProblemShape<ConvOp, SpatialDim>>
inline
get_conv_problem_vector();
@@ -297,7 +297,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kFprop>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 8, 64},
{2, 7, 7, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
@@ -319,7 +319,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kFprop>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 64},
{2, 16, 15, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
@@ -658,7 +658,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 32},
{2, 15, 16, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
@@ -680,7 +680,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 32},
{2, 16, 15, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
@@ -688,6 +688,28 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
{2, 3},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 64, 16, 128}, // nhwc
{640, 1, 1, 128}, // krsc
{0, 0}, // padding lower (pad_h, pad_w)
{0, 0}, // padding upper (pad_h, pad_w)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
1 // group
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 65, 16, 128}, // nhwc
{640, 1, 1, 128}, // krsc
{0, 0}, // padding lower (pad_h, pad_w)
{0, 0}, // padding upper (pad_h, pad_w)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
1 // group
});
return problem_shapes;
}
@@ -751,17 +773,39 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>() {
{2, 2, 3},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 64, 16, 128}, // ndhwc
{640, 1, 1, 1, 128}, // ktrsc
{0, 0, 0}, // padding lower (pad_d, pad_h, pad_w)
{0, 0, 0}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
1 // group
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 65, 16, 128}, // ndhwc
{640, 1, 1, 1, 128}, // ktrsc
{0, 0, 0}, // padding lower (pad_d, pad_h, pad_w)
{0, 0, 0}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
1 // group
});
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Dgrad
// Unit Stride Dgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
// Specialization for 1D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>> inline
get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad>() {
get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>;
std::vector<ProblemShape> problem_shapes;
problem_shapes.push_back({
@@ -884,7 +928,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad>() {
// Specialization for 2D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>> inline
get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad>() {
get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>;
std::vector<ProblemShape> problem_shapes;
problem_shapes.push_back({
@@ -1007,7 +1051,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad>() {
// Specialization for 3D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>> inline
get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad>() {
get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad, false>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>;
std::vector<ProblemShape> problem_shapes;
// Filter-K = 16 for predication
@@ -1082,6 +1126,134 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad>() {
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Strided Dgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
// Specialization for 1D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>> inline
get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, true>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>;
std::vector<ProblemShape> problem_shapes;
// non-packed input/output strides.
// stride divides dilation
// asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 8, 64}, // nqk
{800, 80, 1}, // stride (nqk)
{64, 3, 64}, // ksc
{64, 64, 1}, // stride (ksc)
{800, 80, 1}, // stride (nwc)
{0}, // padding lower (pad_w)
{1}, // padding upper (pad_w)
{2}, // stride (stride_w)
{4}, // dilation (dilation_w)
1 // group
});
// non-packed input/output strides.
// dilation divides stride
// asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 8, 64}, // nqk
{800, 80, 1}, // stride (nqk)
{64, 3, 64}, // ksc
{64, 64, 1}, // stride (ksc)
{800, 80, 1}, // stride (nwc)
{1}, // padding lower (pad_w)
{0}, // padding upper (pad_w)
{4}, // stride (stride_w)
{2}, // dilation (dilation_w)
1 // group
});
// non-packed input/output strides.
// stride dilation dont divide
// asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 8, 64}, // nqk
{800, 80, 1}, // stride (nqk)
{64, 3, 64}, // ksc
{64, 64, 1}, // stride (ksc)
{800, 80, 1}, // stride (nwc)
{1}, // padding lower (pad_w)
{2}, // padding upper (pad_w)
{2}, // stride (stride_w)
{3}, // dilation (dilation_w)
1 // group
});
return problem_shapes;
}
// Specialization for 2D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>> inline
get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, true>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>;
std::vector<ProblemShape> problem_shapes;
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
// mode 0 stride divides dilation
// mode 1 dilation divides stride
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 16, 16, 64},
{256, 2, 5, 64},
{1, 0},
{0, 1},
{2, 4},
{4, 2},
1
});
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
// mode 0 dilation divides stride
// mode 1 stride divides dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 16, 16, 64},
{256, 2, 5, 64},
{1, 0},
{0, 1},
{4, 2},
{2, 4},
1
});
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
// stride dilation dont divide
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 16, 16, 64},
{256, 2, 5, 64},
{1, 0},
{0, 1},
{3, 2},
{2, 3},
1
});
return problem_shapes;
}
// Specialization for 3D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>> inline
get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad, true>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>;
std::vector<ProblemShape> problem_shapes;
// Filter 3x4x5 + asymmetric padding 102/010, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 64},
{64, 3, 4, 5, 96},
{1, 0, 1},
{0, 2, 0},
{2, 4, 2},
{4, 2, 3},
1
});
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass::test
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+8
View File
@@ -93,6 +93,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -137,6 +138,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -181,6 +183,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -225,6 +228,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -273,6 +277,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -317,6 +322,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -361,6 +367,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -405,6 +412,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+8
View File
@@ -93,6 +93,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -137,6 +138,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -181,6 +183,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -225,6 +228,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -273,6 +277,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -317,6 +322,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -361,6 +367,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -405,6 +412,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_tf32_tf32_f32_tensorop_f32.cu
+8
View File
@@ -93,6 +93,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -137,6 +138,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -181,6 +183,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -225,6 +228,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -273,6 +277,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -317,6 +322,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -361,6 +367,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -405,6 +412,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/conv/device_3x/testbed_conv.hpp
+134 -32
View File
@@ -40,6 +40,7 @@
#include "cutlass/kernel_hardware_info.hpp"
#include "cutlass/conv/convolution.h"
#include "cutlass/conv/convnd_problem_shape.hpp"
#include "../test/unit/gemm/device/gemm_testbed_3x.hpp"
#include "thrust/universal_vector.h"
#include "cutlass/util/distribution.h"
@@ -64,6 +65,7 @@
namespace test::conv::device {
/////////////////////////////////////////////////////////////////////////////////////////////////
// Initializes a flat device buffer
template <typename Element>
static void
@@ -104,7 +106,39 @@ initialize_values(
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// utils for sparse or dense conv parameters
template <class Conv>
struct DenseConvParams {
// Default Kernel data types
using ElementA = typename Conv::ConvKernel::ElementA;
using ElementB = typename Conv::ConvKernel::ElementB;
static constexpr cutlass::conv::Operator ConvOp = Conv::DispatchPolicy::ConvOp;
static constexpr int NumSpatialDimensions = Conv::NumSpatialDimensions;
using ProblemShape = cutlass::conv::ConvProblemShape<ConvOp, NumSpatialDimensions>;
// get the default arguments without sparse data
auto get_mainloop_arguments(
ProblemShape const& problem_shape,
thrust::universal_vector<ElementA>& tensor_A,
thrust::universal_vector<ElementB>& tensor_B
) {
auto args = typename Conv::ConvKernel::MainloopArguments {
problem_shape,
tensor_A.data().get(),
tensor_B.data().get(),
};
return args;
}
};
template <class Conv>
struct SparseConvParams {
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <class Conv, bool isSparseEnabled_ = false>
struct ConvTestbed {
// Kernel data types
using ElementA = typename Conv::ConvKernel::ElementA;
@@ -114,6 +148,11 @@ struct ConvTestbed {
using ElementD = typename Conv::ConvKernel::ElementD;
using ElementAccumulator = typename Conv::ConvKernel::ElementAccumulator;
// ConvTest for sparse kernel
static constexpr bool isSparseEnabled = isSparseEnabled_;
using ConvParams = cute::conditional_t<isSparseEnabled, SparseConvParams<Conv>, DenseConvParams<Conv>>;
ConvParams params;
//
// FusionOperation derived types/queries
//
@@ -134,6 +173,8 @@ struct ConvTestbed {
static constexpr bool IsBiasEnabled = cutlass::epilogue::collective::detail::IsThreadEpilogueOpWithBias<FusionOp>::value &&
!cute::is_same_v<BiasType, void>;
static constexpr bool DisableSource = cute::is_void_v<typename FusionOp::ElementSource>;
using StrideC = typename Conv::ConvKernel::StrideC;
using StrideD = typename Conv::ConvKernel::StrideD;
using ThreadEpilogueOp = typename Conv::ConvKernel::CollectiveEpilogue::ThreadEpilogueOp;
@@ -141,6 +182,10 @@ struct ConvTestbed {
static constexpr cutlass::conv::Operator ConvOp = Conv::DispatchPolicy::ConvOp;
static constexpr int NumSpatialDimensions = Conv::NumSpatialDimensions;
using ProblemShape = cutlass::conv::ConvProblemShape<ConvOp, NumSpatialDimensions>;
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
using MaxSwizzleSize = typename gemm::device::detail::MaxSwizzleSize;
using Splits = typename gemm::device::detail::Splits;
using Schedule = typename Conv::DispatchPolicy::Schedule;
/// Initialization
@@ -148,6 +193,7 @@ struct ConvTestbed {
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform;
cutlass::Distribution::Kind init_C = cutlass::Distribution::Uniform;
cutlass::Distribution::Kind init_bias = cutlass::Distribution::Uniform;
cutlass::Distribution::Kind init_disable = cutlass::Distribution::Identity; // all zeros
uint64_t seed = 6090;
float epsilon = 0.0f;
int split_p_slices = 1;
@@ -160,7 +206,8 @@ struct ConvTestbed {
thrust::universal_vector<ElementScalar> tensor_alpha;
thrust::universal_vector<ElementScalar> tensor_beta;
void initialize(ProblemShape const& problem_shape, uint64_t seed = 6090) {
// Return true on success, else false
bool initialize(ProblemShape const& problem_shape, uint64_t seed = 6090) {
tensor_A.resize(sizeof(ElementA) * problem_shape.size_A());
tensor_B.resize(sizeof(ElementB) * problem_shape.size_B());
tensor_C.resize(sizeof(ElementC) * problem_shape.size_C());
@@ -171,6 +218,12 @@ struct ConvTestbed {
initialize_values(tensor_B, init_B, seed * 11);
initialize_values(tensor_C, init_C, seed * 17);
initialize_values(tensor_bias, init_bias, seed * 19);
bool flag = true;
if constexpr (isSparseEnabled) {
flag &= params.initialize(problem_shape, tensor_B, static_cast<int>(seed + 2023));
}
return flag;
}
// Determine SMEM requirements and waive if not satisfied
@@ -190,11 +243,16 @@ struct ConvTestbed {
return max_smem_size >= Conv::ConvKernel::SharedStorageSize;
}
/// Executes one test
// Executes one test
bool run(
ProblemShape const& problem_shape,
ElementScalar alpha = ElementScalar(1),
ElementScalar beta = ElementScalar(0)
,
RasterOrderOptions raster_order = RasterOrderOptions::Heuristic,
MaxSwizzleSize max_swizzle = MaxSwizzleSize{},
Splits splits = Splits{},
DecompositionMode decomposition_mode = DecompositionMode::Heuristic
) {
// Waive test if insufficient CUDA device
@@ -205,7 +263,12 @@ struct ConvTestbed {
return true;
}
initialize(problem_shape);
bool ret = initialize(problem_shape);
if (!ret) {
std::cerr << "initialize failed for the given problem_shape: \n";
return false;
}
cutlass::KernelHardwareInfo hw_info;
cudaGetDevice(&hw_info.device_id);
@@ -230,20 +293,27 @@ struct ConvTestbed {
cute::get<0, i>(stride_D) = problem_shape.stride_C[ProblemShape::RankT-2-i];
});
}
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
typename Conv::ConvKernel::TileScheduler::Arguments scheduler_args{};
if constexpr (cute::is_same_v<typename Conv::ConvKernel::TileScheduler::Arguments, cutlass::gemm::StreamKScheduler>) {
scheduler_args = { static_cast<int>(splits), static_cast<int>(max_swizzle), raster_order, decomposition_mode };
}
auto mainloop_args = params.get_mainloop_arguments(problem_shape, tensor_A, tensor_B);
auto epilogue_args = typename Conv::ConvKernel::EpilogueArguments {
{},
tensor_C.data().get(),
stride_C,
tensor_D_computed.data().get(),
stride_D,
};
auto args = typename Conv::Arguments {
{
problem_shape,
tensor_A.data().get(),
tensor_B.data().get(),
}, // MainloopArguments
{
{},
tensor_C.data().get(),
stride_C,
tensor_D_computed.data().get(),
stride_D,
}, // EpilogueArguments
mainloop_args, // MainloopArguments
epilogue_args, // EpilogueArguments
hw_info,
scheduler_args
};
@@ -462,6 +532,8 @@ struct ConvTestbed {
for (size_t i = 0; i < size_t(size(reference)); ++i) {
if (reference(i) != computed(i)) {
passed = false;
printf("[%llu] %f, %f\n", static_cast<unsigned long long>(i),
float(reference(i)), float(computed(i)));
break;
}
}
@@ -475,6 +547,8 @@ struct ConvTestbed {
if (std::isnan(abs_error) || std::isnan(rel_error) ||
std::min(abs_error, rel_error) > epsilon) {
passed = false;
printf("[%llu] %f, %f\n", static_cast<unsigned long long>(i),
float(reference(i)), float(computed(i)));
break;
}
}
@@ -488,18 +562,20 @@ struct ConvTestbed {
cute::print("\n");
for (size_t i = 0; i < size_t(size(A)); ++i) {
printf("[%ld]: A = %f\n", i, float(A(i)));
printf("[%llu]: A = %f\n", static_cast<unsigned long long>(i), float(A(i)));
}
for (size_t i = 0; i < size_t(size(B)); ++i) {
printf("[%ld]: B = %f\n", i, float(B(i)));
printf("[%llu]: B = %f\n", static_cast<unsigned long long>(i), float(B(i)));
}
if constexpr (IsBiasEnabled) {
for (size_t i = 0; i < size_t(size(tensor_bias)); ++i) {
printf("[%ld]: bias = %f\n", i, float(tensor_bias(i)));
printf("[%llu]: bias = %f\n", static_cast<unsigned long long>(i),
float(tensor_bias(i)));
}
}
for (size_t i = 0; i < size_t(size(reference)); ++i) {
printf("[%ld]: ref = %f, computed = %f\n", i, float(reference(i)), float(computed(i)));
printf("[%llu]: ref = %f, computed = %f\n", static_cast<unsigned long long>(i),
float(reference(i)), float(computed(i)));
}
}
#endif
@@ -509,30 +585,56 @@ struct ConvTestbed {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Conv>
bool TestAllConv(double alpha = 1.0, double beta = 0.0, float epsilon = 0.0f) {
template <typename Conv, bool SupportStrides = (Conv::DispatchPolicy::ConvOp != cutlass::conv::Operator::kDgrad)>
bool TestAllConv(double alpha = 1.0, double beta = 0.0, float epsilon = 0.0f
) {
using ElementScalar = typename Conv::EpilogueOutputOp::ElementScalar;
bool passed = true;
ConvTestbed<Conv> testbed;
testbed.epsilon = epsilon;
auto problem_vector = get_conv_problem_vector<
Conv::NumSpatialDimensions, Conv::DispatchPolicy::ConvOp>();
Conv::NumSpatialDimensions, Conv::DispatchPolicy::ConvOp, SupportStrides>();
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using MaxSwizzleSize = typename gemm::device::detail::MaxSwizzleSize;
using Splits = typename gemm::device::detail::Splits;
std::vector<DecompositionMode> decomposition_modes = {DecompositionMode::Heuristic};
static constexpr bool UsesStreamKScheduler = cute::is_same_v<typename Conv::ConvKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>;
if constexpr (UsesStreamKScheduler) {
decomposition_modes.push_back(DecompositionMode::DataParallel);
decomposition_modes.push_back(DecompositionMode::SplitK);
decomposition_modes.push_back(DecompositionMode::StreamK);
}
for (auto conv_problem : problem_vector) {
#if CUTLASS_DEBUG_TRACE_LEVEL > 0
print(conv_problem);
print(conv_problem);
#endif
for (DecompositionMode decomp_mode : decomposition_modes) {
std::vector problem_splits = {Splits{1}};
if (decomp_mode == DecompositionMode::Heuristic || decomp_mode == DecompositionMode::SplitK) {
problem_splits.push_back(Splits{2});
}
for (auto splits : problem_splits) {
passed = testbed.run(
conv_problem,
cutlass::from_real<ElementScalar>(alpha),
cutlass::from_real<ElementScalar>(beta));
if (!passed) {
printf("Failed test for "); print(conv_problem);
return false;
}
passed = testbed.run(
conv_problem,
cutlass::from_real<ElementScalar>(alpha),
cutlass::from_real<ElementScalar>(beta)
,RasterOrderOptions::Heuristic, // raster_order
MaxSwizzleSize(1),
splits,
decomp_mode
);
if (!passed) {
printf("Failed test for "); print(conv_problem);
return false;
}
} // splits
} // decomposition_mode
}
return passed;
test/unit/core/CMakeLists.txt
+1
View File
@@ -42,6 +42,7 @@ cutlass_test_unit_add_executable(
tensor_view.cu
matrix_coord.cu
numeric_conversion.cu
numeric_conversion_subbyte.cu
fast_numeric_conversion.cu
functional.cu
)
test/unit/core/complex.cu
+373 -2
View File
@@ -32,14 +32,16 @@
\brief CUTLASS host-device template for complex numbers supporting all CUTLASS numeric types.
*/
// Standard Library's std::complex<T> used for reference checking
#include <complex>
#include <cuda/std/complex>
#include "../common/cutlass_unit_test.h"
#include "cutlass/complex.h"
#include "cutlass/constants.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/tfloat32.h"
#include <type_traits>
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -93,7 +95,6 @@ TEST(complex, f16_to_f32_conversion) {
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(complex, exp_f32) {
cutlass::complex<float> Z[] = {
{1, 1},
{2 , cutlass::constants::pi<float>()/2.0f },
@@ -126,6 +127,376 @@ TEST(complex, exp_f32) {
}
}
TEST(complex, absolute_value_real_and_imag) {
{
cutlass::complex z_d{3.0, 4.0};
auto abs_d = cutlass::abs(z_d);
static_assert(std::is_same_v<decltype(abs_d), double>);
EXPECT_EQ(abs_d, 5.0);
auto real_d = cutlass::real(z_d);
static_assert(std::is_same_v<decltype(real_d), double>);
EXPECT_EQ(real_d, 3.0);
auto imag_d = cutlass::imag(z_d);
static_assert(std::is_same_v<decltype(imag_d), double>);
EXPECT_EQ(imag_d, 4.0);
}
{
cutlass::complex z_f{3.0f, 4.0f};
auto abs_f = cutlass::abs(z_f);
static_assert(std::is_same_v<decltype(abs_f), float>);
EXPECT_EQ(abs_f, 5.0f);
auto real_f = cutlass::real(z_f);
static_assert(std::is_same_v<decltype(real_f), float>);
EXPECT_EQ(real_f, 3.0f);
auto imag_f = cutlass::imag(z_f);
static_assert(std::is_same_v<decltype(imag_f), float>);
EXPECT_EQ(imag_f, 4.0f);
}
{
cutlass::complex z_tf32{cutlass::tfloat32_t{3.0f}, cutlass::tfloat32_t{4.0f}};
auto abs_tf32 = cutlass::abs(z_tf32);
static_assert(std::is_same_v<decltype(abs_tf32), cutlass::tfloat32_t>);
EXPECT_EQ(abs_tf32, cutlass::tfloat32_t{5.0f});
auto real_tf32 = cutlass::real(z_tf32);
static_assert(std::is_same_v<decltype(real_tf32), cutlass::tfloat32_t>);
EXPECT_EQ(real_tf32, cutlass::tfloat32_t{3.0f});
auto imag_tf32 = cutlass::imag(z_tf32);
static_assert(std::is_same_v<decltype(imag_tf32), cutlass::tfloat32_t>);
EXPECT_EQ(imag_tf32, cutlass::tfloat32_t{4.0f});
}
{
cutlass::complex z_i{3, 4};
// sqrt(int) isn't a valid overload, so cutlass::abs isn't tested.
auto real_i = cutlass::real(z_i);
static_assert(std::is_same_v<decltype(real_i), int>);
EXPECT_EQ(real_i, 3);
auto imag_i = cutlass::imag(z_i);
static_assert(std::is_same_v<decltype(imag_i), int>);
EXPECT_EQ(imag_i, 4);
}
{
double x_d{3.0};
auto real_d = cutlass::real(x_d);
static_assert(std::is_same_v<decltype(real_d), double>);
EXPECT_EQ(real_d, 3.0);
auto imag_d = cutlass::imag(x_d);
static_assert(std::is_same_v<decltype(imag_d), double>);
EXPECT_EQ(imag_d, 0.0);
}
{
float x_f{3.0f};
auto real_f = cutlass::real(x_f);
static_assert(std::is_same_v<decltype(real_f), float>);
EXPECT_EQ(real_f, 3.0f);
auto imag_f = cutlass::imag(x_f);
static_assert(std::is_same_v<decltype(imag_f), float>);
EXPECT_EQ(imag_f, 0.0f);
}
{
cutlass::tfloat32_t x_tf32{3.0f};
auto real_tf32 = cutlass::real(x_tf32);
static_assert(std::is_same_v<decltype(real_tf32), cutlass::tfloat32_t>);
EXPECT_EQ(real_tf32, cutlass::tfloat32_t{3.0f});
auto imag_tf32 = cutlass::imag(x_tf32);
static_assert(std::is_same_v<decltype(imag_tf32), cutlass::tfloat32_t>);
EXPECT_EQ(imag_tf32, cutlass::tfloat32_t{0.0f});
}
{
int x_i{3};
auto real_i = cutlass::real(x_i);
static_assert(std::is_same_v<decltype(real_i), int>);
EXPECT_EQ(real_i, 3);
auto imag_i = cutlass::imag(x_i);
static_assert(std::is_same_v<decltype(imag_i), int>);
EXPECT_EQ(imag_i, 0);
}
}
// FakeReal and FakeComplex test whether cutlass::real and
// cutlass::imag correctly handle user-defined non-complex
// and complex number types.
namespace test {
// These classes have no conversions to or from arithmetic types, so
// that the test can ensure that the implementation does not silently
// convert to, say, float or int.
class FakeReal {
public:
// cutlass::imag must be able to value-construct its noncomplex input.
FakeReal() = default;
static CUTLASS_HOST_DEVICE FakeReal make_FakeReal(int val) {
return FakeReal{val};
}
friend CUTLASS_HOST_DEVICE bool operator==(FakeReal lhs, FakeReal rhs) {
return lhs.value_ == rhs.value_;
}
friend CUTLASS_HOST_DEVICE FakeReal operator-(FakeReal const& x) {
return make_FakeReal(-x.value_);
}
private:
CUTLASS_HOST_DEVICE FakeReal(int val) : value_(val) {}
int value_ = 0;
};
class FakeComplex {
public:
static CUTLASS_HOST_DEVICE FakeComplex
make_FakeComplex(FakeReal re, FakeReal im) {
return FakeComplex{re, im};
}
// Existence of member functions real and imag tell
// CUTLASS that FakeComplex is a complex number type.
CUTLASS_HOST_DEVICE FakeReal real() const { return real_; }
CUTLASS_HOST_DEVICE FakeReal imag() const { return imag_; }
friend CUTLASS_HOST_DEVICE bool operator==(FakeComplex lhs, FakeComplex rhs) {
return lhs.real_ == rhs.real_ && lhs.imag_ == rhs.imag_;
}
private:
CUTLASS_HOST_DEVICE FakeComplex(FakeReal re, FakeReal im)
: real_(re), imag_(im)
{}
FakeReal real_{};
FakeReal imag_{};
};
CUTLASS_HOST_DEVICE FakeComplex conj(FakeComplex const& z) {
return FakeComplex::make_FakeComplex(z.real(), -z.imag());
}
// Variant of FakeComplex that has a hidden friend conj instead of a
// nonmember conj defined outside the class.
class FakeComplexWithHiddenFriendConj {
public:
static CUTLASS_HOST_DEVICE FakeComplexWithHiddenFriendConj
make_FakeComplexWithHiddenFriendConj(FakeReal re, FakeReal im) {
return FakeComplexWithHiddenFriendConj{re, im};
}
CUTLASS_HOST_DEVICE FakeReal real() const { return real_; }
CUTLASS_HOST_DEVICE FakeReal imag() const { return imag_; }
friend CUTLASS_HOST_DEVICE bool
operator==(FakeComplexWithHiddenFriendConj lhs,
FakeComplexWithHiddenFriendConj rhs)
{
return lhs.real_ == rhs.real_ && lhs.imag_ == rhs.imag_;
}
friend CUTLASS_HOST_DEVICE FakeComplexWithHiddenFriendConj
conj(FakeComplexWithHiddenFriendConj const& z) {
return FakeComplexWithHiddenFriendConj::make_FakeComplexWithHiddenFriendConj(z.real(), -z.imag());
}
private:
CUTLASS_HOST_DEVICE
FakeComplexWithHiddenFriendConj(FakeReal re, FakeReal im)
: real_(re), imag_(im)
{}
FakeReal real_{};
FakeReal imag_{};
};
} // namespace test
TEST(complex, real_and_imag_with_custom_types) {
using test::FakeReal;
using test::FakeComplex;
{
FakeReal x = FakeReal::make_FakeReal(42);
auto x_r = cutlass::real(x);
static_assert(std::is_same_v<decltype(x_r), FakeReal>);
EXPECT_EQ(x_r, FakeReal::make_FakeReal(42));
auto x_i = cutlass::imag(x);
static_assert(std::is_same_v<decltype(x_i), FakeReal>);
EXPECT_EQ(x_i, FakeReal::make_FakeReal(0));
}
{
FakeComplex z = FakeComplex::make_FakeComplex(
FakeReal::make_FakeReal(3), FakeReal::make_FakeReal(4));
auto z_r = cutlass::real(z);
static_assert(std::is_same_v<decltype(z_r), FakeReal>);
EXPECT_EQ(z_r, FakeReal::make_FakeReal(3));
auto z_i = cutlass::imag(z);
static_assert(std::is_same_v<decltype(z_i), FakeReal>);
EXPECT_EQ(z_i, FakeReal::make_FakeReal(4));
}
}
namespace test {
template<class T>
void conj_tester(T z, T z_c_expected, const char type_name[]) {
// Use cutlass::conj just like std::swap (the "std::swap two-step").
using cutlass::conj;
auto z_c = conj(z);
static_assert(std::is_same_v<decltype(z_c), T>);
constexpr bool is_cuComplex = std::is_same_v<T, cuDoubleComplex> ||
std::is_same_v<T, cuFloatComplex>;
if constexpr (is_cuComplex) {
EXPECT_EQ(z_c.x, z_c_expected.x);
EXPECT_EQ(z_c.y, z_c_expected.y) << "conj failed for type " << type_name;
}
else {
EXPECT_EQ(z_c, z_c_expected) << "conj failed for type " << type_name;
}
auto z_c2 = cutlass::conjugate<T>{}(z);
static_assert(std::is_same_v<decltype(z_c2), T>);
if constexpr (is_cuComplex) {
// cuFloatComplex and cuDoubleComplex don't report conj(z) as
// being well-formed, probably because they are type aliases of
// some kind. cutlass::conj works fine, though!
static_assert(! cutlass::platform::is_arithmetic_v<T> &&
(cutlass::detail::has_unqualified_conj_v<T> ||
cutlass::detail::has_cutlass_conj_v<T>));
EXPECT_EQ(z_c2.x, z_c_expected.x);
EXPECT_EQ(z_c2.y, z_c_expected.y)
<< "conjugate failed for type " << type_name;
}
else {
EXPECT_EQ(z_c2, z_c_expected) << "conjugate failed for type " << type_name;
}
}
} // namespace test
TEST(complex, conj_with_standard_arithmetic_types) {
{
double x = 42.0;
double x_c_expected = 42.0;
test::conj_tester(x, x_c_expected, "double");
}
{
float x = 42.0f;
float x_c_expected = 42.0f;
test::conj_tester(x, x_c_expected, "float");
}
{
int x = 42;
int x_c_expected = 42;
test::conj_tester(x, x_c_expected, "int");
}
}
TEST(complex, conj_with_cutlass_complex_types) {
{
cutlass::complex<double> z{3.0, 4.0};
cutlass::complex<double> z_c_expected{3.0, -4.0};
test::conj_tester(z, z_c_expected, "cutlass::complex<double>");
}
{
cutlass::complex<float> z{3.0f, 4.0f};
cutlass::complex<float> z_c_expected{3.0f, -4.0f};
test::conj_tester(z, z_c_expected, "cutlass::complex<float>");
}
{
cutlass::complex<cutlass::tfloat32_t> z{
cutlass::tfloat32_t{3.0f}, cutlass::tfloat32_t{4.0f}};
cutlass::complex<cutlass::tfloat32_t> z_c_expected{
cutlass::tfloat32_t{3.0f}, cutlass::tfloat32_t{-4.0f}};
test::conj_tester(z, z_c_expected, "cutlass::complex<cutlass::tfloat32_t>");
}
}
TEST(complex, conj_with_noncomplex_type_not_in_cutlass_namespace) {
test::FakeReal x = test::FakeReal::make_FakeReal(42);
test::FakeReal x_c_expected = test::FakeReal::make_FakeReal(42);
test::conj_tester(x, x_c_expected, "test::FakeReal");
}
TEST(complex, conj_with_noncomplex_type_in_cutlass_namespace) {
cutlass::tfloat32_t x{42.0f};
cutlass::tfloat32_t x_c_expected{42.0f};
test::conj_tester(x, x_c_expected, "cutlass::tfloat32_t");
}
TEST(complex, conj_with_complex_types_not_in_cutlass_namespace) {
using test::FakeReal;
// conj defined as nonmember outside the class
{
test::FakeComplex z = test::FakeComplex::make_FakeComplex(
FakeReal::make_FakeReal(3), FakeReal::make_FakeReal(4));
test::FakeComplex z_c_expected = test::FakeComplex::make_FakeComplex(
FakeReal::make_FakeReal(3), FakeReal::make_FakeReal(-4));
test::conj_tester(z, z_c_expected, "test::FakeComplex");
}
// conj defined as hidden friend
{
test::FakeComplexWithHiddenFriendConj z =
test::FakeComplexWithHiddenFriendConj::make_FakeComplexWithHiddenFriendConj(
FakeReal::make_FakeReal(3),
FakeReal::make_FakeReal(4));
test::FakeComplexWithHiddenFriendConj z_c_expected =
test::FakeComplexWithHiddenFriendConj::make_FakeComplexWithHiddenFriendConj(
FakeReal::make_FakeReal(3),
FakeReal::make_FakeReal(-4));
test::conj_tester(z, z_c_expected, "test::FakeComplexWithHiddenFriendConj");
}
}
TEST(complex, conj_with_cuda_std_complex_types) {
{
cuda::std::complex<double> z{3.0, 4.0};
cuda::std::complex<double> z_c_expected{3.0, -4.0};
test::conj_tester(z, z_c_expected, "cuda::std::complex<double>");
}
{
cuda::std::complex<float> z{3.0f, 4.0f};
cuda::std::complex<float> z_c_expected{3.0f, -4.0f};
test::conj_tester(z, z_c_expected, "cuda::std::complex<float>");
}
}
TEST(complex, conj_with_cuComplex_types) {
{
cuDoubleComplex z = make_cuDoubleComplex(3.0, 4.0);
cuDoubleComplex z_c_expected = make_cuDoubleComplex(3.0, -4.0);
test::conj_tester(z, z_c_expected, "cuDoubleComplex");
}
{
cuFloatComplex z = make_cuFloatComplex(3.0f, 4.0f);
cuFloatComplex z_c_expected = make_cuFloatComplex(3.0f, -4.0f);
test::conj_tester(z, z_c_expected, "cuFloatComplex");
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
test/unit/core/numeric_conversion.cu
+1
View File
@@ -668,3 +668,4 @@ TYPED_TEST(VectorArrayConverterTest, array_263) {
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/core/numeric_conversion_subbyte.cu
+69
View File
@@ -0,0 +1,69 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Unit tests for conversion operators.
*/
#include "../common/cutlass_unit_test.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/integer_subbyte.h"
namespace test::core::host {
template <class DstValueType, class SrcValueType, int NumElements>
void run_test() {
cutlass::Array<DstValueType, NumElements> dst;
dst.clear();
cutlass::Array<SrcValueType, NumElements> src;
for (int k = 0; k < NumElements; ++k) {
src[k] = SrcValueType(k+1);
}
cutlass::NumericArrayConverter<DstValueType, SrcValueType, NumElements> converter;
dst = converter(src);
for (int k = 0; k < NumElements; ++k) {
EXPECT_TRUE(static_cast<int>(src[k]) == static_cast<int>(dst[k]));
}
}
} // namespace test::core::host
TEST(NumericArrayConversion, Subbyte_int8_int8) {
test::core::host::run_test<int8_t, int8_t, 8>();
}
TEST(NumericArrayConversion, Subbyte_int8_int4) {
test::core::host::run_test<int8_t, cutlass::int4b_t, 8>();
}
test/unit/cute/ampere/CMakeLists.txt
+1
View File
@@ -31,6 +31,7 @@ cutlass_test_unit_add_executable(
cp_async.cu
ldsm.cu
cooperative_gemm.cu
cooperative_copy.cu
)
cutlass_test_unit_add_executable(
test/unit/cute/ampere/cooperative_copy.cu
+633
View File
@@ -0,0 +1,633 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <iostream>
#include <iomanip>
#include <utility>
#include <type_traits>
#include <vector>
#include <numeric>
#include <tuple>
#include <thrust/host_vector.h>
#include <thrust/device_vector.h>
#include <cute/tensor.hpp>
#include <cute/numeric/numeric_types.hpp>
using namespace cute;
namespace cooperative_copy_mode {
struct global_shared {};
struct global_global {};
struct shared_shared {};
}
// gs --> global to/from shared
template <int MaxVecBits, uint32_t ThreadBlockSize, class T, class GMemLayout, class SMemLayout>
__device__ void
cooperative_copy_default_gs(T const* g_in, T* g_out, GMemLayout const& gmem_layout, SMemLayout const& smem_layout)
{
using namespace cute;
extern __shared__ uint128_t smem_buf[];
// Cast smem_buf to smem_uint8_ptr and move it by MaxVecBits bits
// This is to make sure tests pass on pointer aligned to MaxVecBits bits
uint8_t* smem_uint8_ptr = reinterpret_cast<uint8_t*>(smem_buf) + (MaxVecBits/8);
T* smem = reinterpret_cast<T*>(smem_uint8_ptr);
Tensor g_in_tensor = make_tensor(make_gmem_ptr(g_in), gmem_layout);
Tensor g_out_tensor = make_tensor(make_gmem_ptr(g_out), gmem_layout);
Tensor s_tensor = make_tensor(make_smem_ptr(smem), smem_layout);
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, g_in_tensor, s_tensor);
cp_async_fence();
cp_async_wait<0>();
__syncthreads();
if(thread0()) {
for(int i = 0; i < size(s_tensor); ++i) {
s_tensor(i) += T(i);
}
}
__syncthreads();
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, s_tensor, g_out_tensor);
}
// ss --> shared to shared
template <int MaxVecBits, uint32_t ThreadBlockSize, class T, class Layout1, class Layout2>
__device__ void
cooperative_copy_default_ss(T const* g_in, T* g_out, Layout1 const& layout1, Layout2 const& layout2)
{
using namespace cute;
extern __shared__ uint128_t smem_buf[];
// Cast smem_buf to smem_uint8_ptr and move it by MaxVecBits bits
// This is to make sure tests pass on pointer aligned to MaxVecBits bits
T* smem1 = reinterpret_cast<T*>(smem_buf);
uint8_t* smem2_uint8_ptr = reinterpret_cast<uint8_t*>(smem_buf) + (MaxVecBits/8);
T* smem2 = reinterpret_cast<T*>(smem2_uint8_ptr) + cute::cosize(layout2);
Tensor g_in_tensor = make_tensor(make_gmem_ptr(g_in), layout1);
Tensor g_out_tensor = make_tensor(make_gmem_ptr(g_out), layout2);
Tensor s1_tensor = make_tensor(make_smem_ptr(smem1), layout2);
Tensor s2_tensor = make_tensor(make_smem_ptr(smem2), layout1);
cooperative_copy<ThreadBlockSize, cute::sizeof_bits_v<T>>(threadIdx.x, g_in_tensor, s1_tensor);
cp_async_fence();
cp_async_wait<0>();
__syncthreads();
if(thread0()) {
for(int i = 0; i < size(s1_tensor); ++i) {
s1_tensor(i) += T(i);
}
}
__syncthreads();
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, s1_tensor, s2_tensor);
__syncthreads();
cooperative_copy<ThreadBlockSize, cute::sizeof_bits_v<T>>(threadIdx.x, s2_tensor, g_out_tensor);
}
// gg --> global to global
template <int MaxVecBits, uint32_t ThreadBlockSize, class T, class Layout1, class Layout2>
__device__ void
cooperative_copy_default_gg(T const* g_in, T* g_out, Layout1 const& layout1, Layout2 const& layout2)
{
using namespace cute;
Tensor g_in_tensor = make_tensor(make_gmem_ptr(g_in), layout1);
Tensor g_out_tensor = make_tensor(make_gmem_ptr(g_out), layout2);
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, g_in_tensor, g_out_tensor);
}
template <class Mode, int MaxVecBits, uint32_t ThreadBlockSize, class T, class Layout1, class Layout2>
__global__ void
cooperative_copy_default_kernel(T const* g_in, T* g_out, Layout1 const layout1, Layout2 const layout2)
{
if constexpr(std::is_same_v<Mode, cooperative_copy_mode::global_shared>) {
cooperative_copy_default_gs<MaxVecBits, ThreadBlockSize>(g_in, g_out, layout1, layout2);
} else if constexpr (std::is_same_v<Mode, cooperative_copy_mode::global_global>) {
cooperative_copy_default_gg<MaxVecBits, ThreadBlockSize>(g_in, g_out, layout1, layout2);
} else if constexpr (std::is_same_v<Mode, cooperative_copy_mode::shared_shared>) {
cooperative_copy_default_ss<MaxVecBits, ThreadBlockSize>(g_in, g_out, layout1, layout2);
}
}
// Mode - defines memory types of src and dst in cooperative_copy operation
// MaxVecBits - defines max vectorization in cooperative_copy operation, and enforces that
// alignment on used pointers to ensure correct testing
template <class Mode, int MaxVecBits, uint32_t ThreadBlockSize, class T, class Layout1, class Layout2>
void test_cooperative_copy_default(Layout1 const& layout1, Layout2 const& layout2)
{
using value_type = T;
CUTE_STATIC_ASSERT_V(cute::size(layout1) == cute::size(layout2));
auto gmem_layout_in = layout1;
auto gmem_layout_out = cute::conditional_return<std::is_same_v<Mode, cooperative_copy_mode::global_shared>>(layout1, layout2);
#if 0
print(" "); print("layout1: "); print(layout1); print("\n");
print(" "); print("layout2: "); print(layout2); print("\n");
print(" "); print("threads: "); print(ThreadBlockSize); print("\n");
print(" "); print("maxvecbits: "); print(MaxVecBits); print("\n");
#endif
if constexpr (MaxVecBits < cute::sizeof_bits_v<value_type>) {
GTEST_SKIP() << "Skipping test since MaxVecBits (=" << MaxVecBits
<< ") < cute::sizeof_bits_v<value_type> (=" << cute::sizeof_bits_v<value_type> << ")";
} else {
constexpr auto max_vec_bytes = MaxVecBits / 8;
static_assert((max_vec_bytes % sizeof(T)) == 0);
uint32_t count = cute::cosize(gmem_layout_in);
// Extra elements to force MaxVecBits alignment in global memory
uint32_t extra_elements = max_vec_bytes / sizeof(value_type);
// Allocate
thrust::host_vector<value_type> h_in (count + extra_elements);
thrust::host_vector<value_type> h_out(count + extra_elements);
// Initialize
Tensor h_in_tensor = make_tensor(h_in.data() + extra_elements, gmem_layout_in);
Tensor h_out_tensor = make_tensor(h_out.data() + extra_elements, gmem_layout_out);
for (int i = 0; i < cute::size(h_in_tensor); ++i) {
h_in_tensor(i) = value_type(float(i));
// For global-to-global copy need to compare against the same value
h_out_tensor(i) = std::is_same_v<Mode, cooperative_copy_mode::global_global> ? value_type(float(i)) : value_type(float(2 * i));
}
// To GPU
thrust::device_vector<value_type> d_in = h_in;
thrust::device_vector<value_type> d_out(d_in.size(), value_type(float(-2)));
// Adds (MaxVecBits/8) bytes to shared memory as we'll move pointer by that many bytes inside the kernel to enforce
// alignment to (MaxVecBits/8) bytes
size_t shared_memory_bytes = (sizeof(value_type) * count) + max_vec_bytes;
shared_memory_bytes += std::is_same_v<Mode, cooperative_copy_mode::shared_shared> * (sizeof(value_type) * count);
// Launch
auto coop_copy = cooperative_copy_default_kernel<Mode, MaxVecBits, ThreadBlockSize, value_type, Layout1, Layout2>;
ASSERT_EQ(cudaFuncSetAttribute(coop_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(shared_memory_bytes)), cudaSuccess);
auto d_in_ptr = thrust::raw_pointer_cast(d_in.data() + extra_elements);
auto d_out_ptr = thrust::raw_pointer_cast(d_out.data() + extra_elements);
coop_copy<<<1, ThreadBlockSize, shared_memory_bytes>>>(d_in_ptr, d_out_ptr, layout1, layout2);
cudaError_t result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
cudaError_t error = cudaGetLastError();
FAIL() << "Error at kernel sync: " << cudaGetErrorString(error) << "\n";
}
// Validate
thrust::host_vector<value_type> h_result = d_out;
Tensor h_result_tensor = make_tensor(h_result.data() + extra_elements, gmem_layout_out);
for (int i = 0; i < cute::size(h_in_tensor); ++i) {
ASSERT_EQ(h_result_tensor(i), h_out_tensor(i))
<< i << " - result:" << h_result_tensor(i) << " expected:" << h_out_tensor(i);
}
}
}
template<class T>
class SM80_CuTe_Ampere;
template<class Mode, class MaxVecBits>
class SM80_CuTe_Ampere<std::tuple<Mode, MaxVecBits>>: public testing::Test
{
public:
using mode = Mode;
static constexpr int max_vec_bits = MaxVecBits::value;
};
typedef testing::Types<
std::tuple<cooperative_copy_mode::global_shared, cute::Int<128>>,
std::tuple<cooperative_copy_mode::global_shared, cute::Int<64>>,
std::tuple<cooperative_copy_mode::global_shared, cute::Int<32>>,
std::tuple<cooperative_copy_mode::global_shared, cute::Int<16>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<128>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<64>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<32>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<16>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<128>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<64>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<32>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<16>>,
> CooperativeCopyModeMaxVecBitsList;
TYPED_TEST_SUITE(SM80_CuTe_Ampere, CooperativeCopyModeMaxVecBitsList);
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault1D)
{
using value_type = float;
constexpr uint32_t count = 512;
auto gmem_layout = make_layout(make_shape(Int<count>{}));
auto smem_layout = make_layout(make_shape(Int<count>{}));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault1DFallback)
{
using value_type = float;
constexpr uint32_t count = 99;
auto gmem_layout = make_layout(make_shape(Int<count>{}));
auto smem_layout = make_layout(make_shape(Int<count>{}));
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault2D)
{
using value_type = float;
constexpr uint32_t x = 32;
constexpr uint32_t y = 32;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}));
auto smem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
#if 0
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault2DDynamicStrides)
{
using value_type = float;
constexpr uint32_t x = 32;
constexpr uint32_t y = 32;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(1, x));
auto smem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(1, x));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault2DMixedStrides)
{
using value_type = float;
constexpr uint32_t x = 32;
constexpr uint32_t y = 32;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}));
auto smem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(1, x));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
#endif
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault2DFallback)
{
using value_type = float;
constexpr uint32_t x = 37;
constexpr uint32_t y = 37;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}));
auto smem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast Path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault2DCustomStride)
{
using value_type = float;
constexpr uint32_t x = 16;
constexpr uint32_t y = 16;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(Int<y>{}, Int<1>{}));
auto smem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(Int<1>{}, Int<x>{}));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault3D)
{
using value_type = cute::half_t;
constexpr uint32_t x = 8;
constexpr uint32_t y = 8;
constexpr uint32_t z = 16;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{}));
auto smem_layout = make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{}));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefault2Dto3D)
{
using value_type = double;
constexpr uint32_t x = 16;
constexpr uint32_t y = 16;
constexpr uint32_t z = 4;
auto gmem_layout = make_layout(make_shape(Int<x>{}, Int<y*z>{}));
auto smem_layout = make_layout(make_shape(Int<z>{}, Int<y>{}, Int<x>{}));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultCustom1)
{
using value_type = double;
auto gmem_layout = make_layout(
make_shape(Int<8>{}, make_shape(Int<2>{}, Int<2>{})),
make_stride(Int<2>{}, make_shape(Int<1>{}, Int<16>{}))
);
auto smem_layout = make_layout(
make_shape(Int<8>{}, Int<4>{}),
make_stride(Int<4>{}, Int<1>{})
);
constexpr uint32_t thread_block_size = 8;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast Path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultCustom2)
{
using value_type = float;
auto gmem_layout = make_layout(
make_shape(make_shape(Int<4>{}, Int<2>{}), make_shape(Int<2>{}, Int<2>{})),
make_stride(make_shape(Int<4>{}, Int<1>{}), make_shape(Int<16>{}, Int<2>{}))
);
auto smem_layout = make_layout(
make_shape(make_shape(Int<2>{}, Int<2>{}, Int<2>{}), make_shape(Int<2>{}, Int<2>{})),
make_stride(make_shape(Int<16>{}, Int<4>{}, Int<1>{}), make_shape(Int<8>{}, Int<2>{}))
);
constexpr uint32_t thread_block_size = 16;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast Path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzle1)
{
using value_type = float;
auto gmem_layout = Layout<Shape<_8, _64>, Stride<_64, _1>>{};
auto smem_layout = composition(Swizzle<3, 3, 3>{}, Layout<Shape<_8, _64>, Stride<_64, _1>>{});
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast Path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzle2)
{
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<64>{}, Int<64>{}));
auto smem_atom_layout = composition(Swizzle<3, 2, 3>{}, Layout<Shape<_8, _32>, Stride<_32, _1>>{});
auto smem_layout = tile_to_shape(
smem_atom_layout,
make_shape(shape<0>(gmem_layout), shape<1>(gmem_layout))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast Path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzle3)
{
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<64>{}, Int<64>{}));
auto smem_atom_layout = composition(Swizzle<2, 4, 3>{}, Layout<Shape<_16, _64>, Stride<_64, _1>>{});
auto smem_layout = tile_to_shape(
smem_atom_layout,
make_shape(shape<0>(gmem_layout), shape<1>(gmem_layout))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzle4)
{
using value_type = cute::half_t;
auto gmem_atom_layout = composition(Swizzle<3, 2, 3>{}, Layout<Shape<_8, _32>, Stride<_32, _1>>{});
auto smem_layout = make_layout(make_shape(Int<64>{}, Int<64>{}));
auto gmem_layout = tile_to_shape(
gmem_atom_layout,
make_shape(shape<0>(smem_layout), shape<1>(smem_layout))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Needs coalescing to work on fast path
// OK if we enforce slow path
// Problem: Wrong condition when we select between slow and fast path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultCoalesceToCompose)
{
constexpr int m = 96;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<m>{}), GenColMajor{});
auto smem_layout = make_layout(make_shape(Int<m>{}, Int<m>{}), GenColMajor{});
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// Fast path (default): OK
// Slow path (enforced): OK
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzle5)
{
constexpr int m = 64;
constexpr int n = 128;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenColMajor{});
// auto smem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenColMajor{}));
auto smem_atom_layout =
composition(Swizzle<3,3,3>{},
Layout<Shape < _8,_64>,
Stride<_64, _1>>{});
auto smem_layout = tile_to_shape(
smem_atom_layout,
make_shape(shape<0>(gmem_layout), shape<1>(gmem_layout))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// If condition not strict enought will go to fast path
// This test needs checking if CuTe can compose layouts
// Fast path (default): fail
// Slow path (enforced): Should go to vectorized naive path
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzleNaiveVectorizable)
{
constexpr int m = 192;
constexpr int n = 64;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenColMajor{});
// auto smem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenColMajor{});
auto smem_atom_layout =
composition(Swizzle<3,3,3>{},
Layout<Shape <_64, _8>,
Stride< _1,_64>>{});
auto smem_layout = tile_to_shape(
smem_atom_layout,
shape(gmem_layout)
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// fast path: ok (chosen)
// slow path: ok
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultRowMajorSmall)
{
constexpr int m = 24;
constexpr int n = 8;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
auto smem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// fast path: doesn't apply
// slow path: ok
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSlowPath)
{
constexpr int m = 67;
constexpr int n = 67;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
auto smem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
// fast path: doesn't apply
// slow path: should vectorize
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopyDefaultSwizzleSlowPathVectorize)
{
constexpr int m = 68;
constexpr int n = 68;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
auto smem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
constexpr uint32_t thread_block_size = 32;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
TYPED_TEST(SM80_CuTe_Ampere, CooperativeCopy48x48Swizzle)
{
constexpr int m = 48;
constexpr int n = 48;
using value_type = cute::half_t;
auto gmem_layout = make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{});
auto smem_layout = composition(Swizzle<2,2,3>{},
Layout<Shape <Shape <_16, _3, Int<48>>>,
Stride<Stride< _1, Int<768>, _16>>>{});
constexpr uint32_t thread_block_size = 8 * 32;
test_cooperative_copy_default<cooperative_copy_mode::shared_shared,
TestFixture::max_vec_bits,
thread_block_size,
value_type>(gmem_layout, smem_layout);
}
test/unit/cute/ampere/cooperative_gemm.cu
+143
View File
@@ -298,3 +298,146 @@ TEST(SM80_CuTe_Ampere, CooperativeGemm8_MixedPrecisionTF32FP32_MMA) {
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 128, TA, TB, TC>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm9_C64C64C64_MMA) {
using TA = cutlass::complex<double>;
using TB = cutlass::complex<double>;
using TC = cutlass::complex<double>;
constexpr uint32_t thread_block_size = 256;
constexpr int MaxVecBits = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_8x8x4_C64C64C64C64_TN>,
Layout<Shape<_4, _4, _1>, Stride<_1, _4, _0>>,
Tile<Underscore, Underscore, Underscore>
>;
using ALayout = Layout<Shape<Int<13>,Int<35>>, Stride<Int<44>, Int<1> >>;
using BLayout = Layout<Shape< Int<7>, Int<35>>, Stride<Int<44>, Int<1> >>;
using CLayout = Layout<Shape<Int<13>, Int<7>>, Stride< Int<1>, Int<30>>>;
test_cooperative_gemm<ALayout,
BLayout,
CLayout,
ALayout,
BLayout,
CLayout,
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // A
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // B
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // C
thread_block_size,
tiled_mma_t,
MaxVecBits,
TA,
TB,
TC>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm10_F16F64F16_FMA) {
using TA = cutlass::half_t;
using TB = double;
using TC = cutlass::half_t;
constexpr uint32_t thread_block_size = 256;
constexpr int MaxVecBits = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<UniversalFMA<half_t, half_t, double, half_t>>,
Layout<Shape<_16, _16, _1>, Stride<_1, _16, _0>>,
Tile<Underscore, Underscore, Underscore>
>;
using ALayout = Layout<Shape<Int<64>,Int<64>>, Stride<Int<64>, Int< 1>>>;
using BLayout = Layout<Shape<Int<64>,Int<64>>, Stride<Int< 1>, Int<64>>>;
using CLayout = Layout<Shape<Int<64>,Int<64>>, Stride<Int< 1>, Int<64>>>;
test_cooperative_gemm<ALayout,
BLayout,
CLayout,
ALayout,
BLayout,
CLayout,
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // A
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // B
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // C
thread_block_size,
tiled_mma_t,
MaxVecBits,
TA,
TB,
TC>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemmComposedStride) {
using T = cute::half_t;
constexpr uint32_t thread_block_size = 128;
constexpr int MaxVecBits = 16;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x16_F16F16F16F16_TN>,
Layout<Shape<_2, _2, _1>, Stride<_1, _2, _0>>,
Tile<Underscore, Underscore, Underscore>
>;
using swizzle = cute::Swizzle<3, 3, 3>;
using offset = cute::_0;
using atom_tile_right = decltype(cute::make_layout(cute::Shape<cute::_8, cute::_64>{}, cute::LayoutRight{}));
using FP16AtomLayoutRight = decltype(cute::composition(swizzle{}, offset{}, atom_tile_right{}));
using shape = cute::Shape<cute::Int<128>, cute::Int<128>>;
using global_a_layout = decltype(cute::make_layout(shape{}, cute::LayoutRight{}));
using global_b_layout = decltype(cute::make_layout(shape{}, cute::LayoutLeft{}));
using global_c_layout = decltype(cute::make_layout(shape{}, cute::LayoutRight{}));
// This is for A row major, B col major according to CUTLASS default configs
using ALayout = decltype(cute::tile_to_shape(FP16AtomLayoutRight{}, global_a_layout{}));
using BLayout = decltype(cute::tile_to_shape(FP16AtomLayoutRight{}, global_b_layout{}));
using CLayout = global_c_layout;
test_cooperative_gemm<ALayout,
BLayout,
CLayout,
ALayout,
BLayout,
CLayout,
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // A
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // B
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // C
thread_block_size,
tiled_mma_t,
MaxVecBits,
T,
T,
T>();
}
TEST(SM89_CuTe_Ampere, CooperativeGemm8_MixedPrecisionTF32FP32_Transform) {
using TA = cutlass::tfloat32_t;
using TB = cutlass::tfloat32_t;
using TC = float;
constexpr uint32_t m = 9;
constexpr uint32_t n = 9;
constexpr uint32_t k = 9;
constexpr uint32_t thread_block_size = 64;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x8_F32TF32TF32F32_TN>,
Layout<Shape<_1, _2, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 16, TA, TB, TC>(cute::negate{}, cute::negate{}, cute::negate{}, cute::negate{});
}
test/unit/cute/cooperative_gemm_common.hpp
+27 -9
View File
@@ -31,6 +31,7 @@
#pragma once
#include "cutlass/relatively_equal.h"
#include "cutlass_unit_test.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -43,6 +44,16 @@
using namespace cute;
template<typename T>
struct fp64_tester {
using value_type = double;
};
template<typename T>
struct fp64_tester<complex<T>> {
using value_type = complex<double>;
};
template<class ALayout,
class BLayout,
class CLayout,
@@ -146,6 +157,11 @@ void test_cooperative_gemm(ALoadTransform const& a_load_transform = {},
using smem_b_layout_t = SMemBLayout;
using smem_c_layout_t = SMemCLayout;
static_assert(std::is_same_v<typename fp64_tester<TA>::value_type, typename fp64_tester<TB>::value_type>);
static_assert(std::is_same_v<typename fp64_tester<TB>::value_type, typename fp64_tester<TC>::value_type>);
using tester = fp64_tester<TA>;
using ABC_64 = typename tester::value_type;
static_assert(size<0>(gmem_a_layout_t{}) == size<0>(gmem_c_layout_t{})); // AM == CM
static_assert(size<0>(gmem_b_layout_t{}) == size<1>(gmem_c_layout_t{})); // BN == CN
static_assert(size<1>(gmem_a_layout_t{}) == size<1>(gmem_b_layout_t{})); // AK == BK
@@ -184,7 +200,7 @@ void test_cooperative_gemm(ALoadTransform const& a_load_transform = {},
h_a_tensor(i) = static_cast<TA>(di / size(gmem_a_layout_t{}));
}
if(i < size(gmem_b_layout_t{})) {
h_b_tensor(i) = static_cast<TA>(di / size(gmem_a_layout_t{}));
h_b_tensor(i) = static_cast<TB>(di / size(gmem_a_layout_t{}));
}
if(i < size(gmem_c_layout_t{})) {
h_c_tensor(i) = static_cast<TC>((di*di) / size(gmem_a_layout_t{}));
@@ -196,8 +212,10 @@ void test_cooperative_gemm(ALoadTransform const& a_load_transform = {},
thrust::device_vector<TC> d_c(h_c);
thrust::device_vector<TC> d_c_out(h_c_out.size(), TC(float(-1)));
const size_t shared_memory_size =
(sizeof(TA) * h_a.size()) + (sizeof(TB) * h_b.size()) + (sizeof(TC) * h_c.size());
constexpr uint32_t copy_max_vec_bytes = CopyMaxVecBits / 8;
const size_t shared_memory_size = round_up(sizeof(TA) * h_a.size(), copy_max_vec_bytes)
+ round_up(sizeof(TB) * h_b.size(), copy_max_vec_bytes)
+ (sizeof(TC) * h_c.size());
auto kernel = cooperative_gemm_kernel<
gmem_a_layout_t, gmem_b_layout_t, gmem_c_layout_t,
smem_a_layout_t, smem_b_layout_t, smem_c_layout_t,
@@ -234,24 +252,24 @@ void test_cooperative_gemm(ALoadTransform const& a_load_transform = {},
for (int n = 0; n < size<0>(h_b_tensor); n++) {
const auto a_value = a_load_transform(h_a_tensor(m, k));
const auto b_value = b_load_transform(h_b_tensor(n, k));
const auto a_value_fp64 = static_cast<double>(a_value);
const auto b_value_fp64 = static_cast<double>(b_value);
const auto a_value_fp64 = static_cast<ABC_64>(a_value);
const auto b_value_fp64 = static_cast<ABC_64>(b_value);
h_c_ref_tensor(m, n) += static_cast<TC>(a_value_fp64 * b_value_fp64);
}
}
}
// C = A*B + C
for (int i = 0; i < size(h_c_ref_tensor); i++) {
const auto ab_value_fp64 = static_cast<double>(h_c_ref_tensor(i));
const auto c_value_fp64 = static_cast<double>(c_load_transform(h_c_tensor(i)));
const auto ab_value_fp64 = static_cast<ABC_64>(h_c_ref_tensor(i));
const auto c_value_fp64 = static_cast<ABC_64>(c_load_transform(h_c_tensor(i)));
h_c_ref_tensor(i) = c_store_transform(static_cast<TC>(alpha * ab_value_fp64 + beta * c_value_fp64));
}
h_c_out = d_c_out;
auto h_c_out_tensor = make_tensor(h_c_out.data(), gmem_c_layout_t{});
for (int i = 0; i < size(h_c_ref_tensor); i++) {
double h_c_ref_i = h_c_ref_tensor(i);
double h_c_out_i = h_c_out_tensor(i);
ABC_64 h_c_ref_i = h_c_ref_tensor(i);
ABC_64 h_c_out_i = h_c_out_tensor(i);
double epsilon(0.1f);
double nonzero_floor(std::numeric_limits<double>::min());
bool passed = cutlass::relatively_equal(h_c_out_i, h_c_ref_i, epsilon, nonzero_floor);
test/unit/cute/core/CMakeLists.txt
+4 -1
View File
@@ -38,16 +38,19 @@ cutlass_test_unit_add_executable(
composition.cpp
constants.cpp
core_unit.cpp
domain_distribute.cpp
inverse_left.cpp
inverse_right.cpp
logical_divide.cpp
logical_product.cpp
math.cpp
math.cpp
mixedbits.cpp
nullspace.cpp
packed_tuple.cpp
pointer.cpp
reverse.cpp
transform.cpp
tuple.cpp
tuple_find.cpp
int_tuple.cpp
)
test/unit/cute/core/array_subbyte.cpp
+9 -9
View File
@@ -51,7 +51,7 @@ TEST(CuTe_core, ArraySubbyte)
for (size_t i = 0; i < array1.size(); ++i) {
array0[i+5] = array1[i];
}
EXPECT_EQ(int4_t(array0.back()), int4_t(1));
for (size_t i = 0; i < array1.size(); ++i) {
@@ -137,7 +137,7 @@ TEST(CuTe_core, Subbyte_iterator)
{
array_subbyte<uint8_t, 15> a{};
auto tensor = make_tensor(subbyte_iterator<uint8_t>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, uint8_t(13));
for (int i = 0; i < int(a.size()); ++i) {
@@ -150,7 +150,7 @@ TEST(CuTe_core, Subbyte_iterator)
{
array_subbyte<int4_t, 15> a{};
auto tensor = make_tensor(subbyte_iterator<int4_t>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, int4_t(-5));
for (int i = 0; i < int(a.size()); ++i) {
@@ -163,7 +163,7 @@ TEST(CuTe_core, Subbyte_iterator)
{
array_subbyte<uint2_t, 15> a{};
auto tensor = make_tensor(subbyte_iterator<uint2_t>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, uint2_t(-5));
for (int i = 0; i < int(a.size()); ++i) {
@@ -176,7 +176,7 @@ TEST(CuTe_core, Subbyte_iterator)
{
array_subbyte<bool, 15> a{};
auto tensor = make_tensor(subbyte_iterator<bool>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, bool(1));
for (int i = 0; i < int(a.size()); ++i) {
@@ -193,7 +193,7 @@ TEST(CuTe_core, Const_subbyte_iterator)
{
array_subbyte<uint8_t, 15> a{};
auto tensor = make_tensor(subbyte_iterator<uint8_t const>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, uint8_t(13));
for (int i = 0; i < int(a.size()); ++i) {
@@ -206,7 +206,7 @@ TEST(CuTe_core, Const_subbyte_iterator)
{
array_subbyte<int4_t, 15> a{};
auto tensor = make_tensor(subbyte_iterator<int4_t const>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, int4_t(-5));
for (int i = 0; i < int(a.size()); ++i) {
@@ -219,7 +219,7 @@ TEST(CuTe_core, Const_subbyte_iterator)
{
array_subbyte<uint2_t, 15> a{};
auto tensor = make_tensor(subbyte_iterator<uint2_t const>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, uint2_t(-5));
for (int i = 0; i < int(a.size()); ++i) {
@@ -232,7 +232,7 @@ TEST(CuTe_core, Const_subbyte_iterator)
{
array_subbyte<bool, 15> a{};
auto tensor = make_tensor(subbyte_iterator<bool const>(a.raw_data()), make_shape(15));
auto tensor = make_tensor(a.begin(), make_shape(15));
fill(a, bool(1));
for (int i = 0; i < int(a.size()); ++i) {
test/unit/cute/core/domain_distribute.cpp
+109
View File
@@ -0,0 +1,109 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <iostream>
#include <cute/tensor.hpp>
using namespace cute;
template <class LayoutA, class LayoutB>
void
test_distribute(LayoutA const& layoutA,
LayoutB const& layoutB)
{
auto layoutR = domain_distribute(shape(layoutA), shape(layoutB));
CUTLASS_TRACE_HOST("test_distribute()");
CUTLASS_TRACE_HOST(layoutA << " <-> " << layoutB);
CUTLASS_TRACE_HOST(" => ");
CUTLASS_TRACE_HOST(layoutR);
// Test that layout B is softly compatible with layout R
EXPECT_TRUE(softly_compatible(layoutB, layoutR));
// Post-condition on the codomain of the distribute
for (int i = 0; i < size(layoutR); ++i) {
for (int j = i+1; j < size(layoutR); ++j) {
EXPECT_TRUE(layoutR(i) < layoutR(j)); // Surjective and Ordered
}
}
}
TEST(CuTe_core, Distribute)
{
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("DOMAIN DISTRIBUTE" );
CUTLASS_TRACE_HOST("-------------------------------");
{
auto shape_a = Shape<Shape<_64,_3>,Shape<_8,_8>>{};
auto shape_b = _128{};
test_distribute(shape_a, shape_b);
}
{
auto shape_a = Shape<Int<192>,Shape<_8,_8>>{};
auto shape_b = _128{};
test_distribute(shape_a, shape_b);
}
{
auto shape_a = Shape<Shape<_64,_3>,Shape<_8,_8>>{};
auto shape_b = _128{} * _8{};
test_distribute(shape_a, shape_b);
}
{
auto shape_a = Shape<Int<192>,Shape<_8,_8>>{};
auto shape_b = _128{} * _8{};
test_distribute(shape_a, shape_b);
}
{
auto shape_a = Shape<Shape<_64,_3>>{};
auto shape_b = _128{};
test_distribute(shape_a, shape_b);
}
}
test/unit/cute/core/int_tuple.cpp
+51 -4
View File
@@ -56,7 +56,7 @@ TEST(CuTe_core, WeaklyCongruent)
EXPECT_TRUE (weakly_congruent(a0, a0));
EXPECT_TRUE (weakly_congruent(b0, b0));
EXPECT_TRUE (weakly_congruent(a0, b0));
auto a1 = Shape<_1, _1>{};
EXPECT_TRUE (weakly_congruent(a , a1));
EXPECT_FALSE(weakly_congruent(a0, a1));
@@ -93,7 +93,7 @@ TEST(CuTe_core, WeaklyCompatible)
EXPECT_TRUE (weakly_compatible(a, a));
EXPECT_TRUE (weakly_compatible(b, b));
EXPECT_TRUE (weakly_compatible(c, c));
EXPECT_FALSE(weakly_compatible(a, b));
EXPECT_FALSE(weakly_compatible(a, b));
EXPECT_FALSE(weakly_compatible(a, c));
EXPECT_TRUE (weakly_compatible(c, a));
@@ -102,9 +102,9 @@ TEST(CuTe_core, WeaklyCompatible)
EXPECT_TRUE (weakly_compatible(a , a0));
EXPECT_FALSE(weakly_compatible(a0, a ));
EXPECT_TRUE (weakly_compatible(c , a0));
EXPECT_FALSE(weakly_compatible(a0, c ));
EXPECT_FALSE(weakly_compatible(a0, c ));
EXPECT_FALSE(weakly_compatible(b , a0));
EXPECT_FALSE(weakly_compatible(a0, b ));
EXPECT_FALSE(weakly_compatible(a0, b ));
auto a1 = Shape<_2,_8>{};
EXPECT_TRUE (weakly_compatible(a1, a1));
@@ -129,3 +129,50 @@ TEST(CuTe_core, WeaklyCompatible)
EXPECT_TRUE (weakly_compatible(a2, a3));
EXPECT_FALSE(weakly_compatible(a3, a2));
}
TEST(CuTe_core, SoftlyCompatible)
{
using namespace cute;
auto a = _16{};
auto b = _12{};
auto c = _8{};
EXPECT_TRUE (softly_compatible(a, a));
EXPECT_TRUE (softly_compatible(b, b));
EXPECT_TRUE (softly_compatible(c, c));
EXPECT_FALSE(softly_compatible(a, b));
EXPECT_TRUE (softly_compatible(a, c));
EXPECT_FALSE(softly_compatible(c, a));
auto a0 = Shape<_16>{};
EXPECT_TRUE (softly_compatible(a0, a0));
EXPECT_TRUE (softly_compatible(a , a0));
EXPECT_FALSE(softly_compatible(a0, a ));
EXPECT_FALSE(softly_compatible(c , a0));
EXPECT_FALSE(softly_compatible(a0, c ));
EXPECT_FALSE(softly_compatible(b , a0));
EXPECT_FALSE(softly_compatible(a0, b ));
auto a1 = Shape<_2,_8>{};
EXPECT_TRUE (softly_compatible(a1, a1));
EXPECT_TRUE (softly_compatible(a , a1));
EXPECT_FALSE(softly_compatible(a0, a1));
EXPECT_FALSE(softly_compatible(a1, a0));
EXPECT_TRUE (softly_compatible(a1, Shape<_2,Shape<_2,_4>>{}));
auto a2 = Shape<Shape<_2,_8>>{};
EXPECT_TRUE (softly_compatible(a2, a2));
EXPECT_TRUE (softly_compatible(a , a2));
EXPECT_FALSE(softly_compatible(c , a2));
EXPECT_TRUE (softly_compatible(a0, a2));
EXPECT_FALSE(softly_compatible(a2, a0));
auto a3 = Shape<Shape<_2,Shape<_4,_2>>>{};
EXPECT_TRUE (softly_compatible(a3, a3));
EXPECT_TRUE (softly_compatible(a , a3));
EXPECT_FALSE(softly_compatible(c , a3));
EXPECT_TRUE (softly_compatible(a0, a3));
EXPECT_FALSE(softly_compatible(a3, a0));
EXPECT_TRUE (softly_compatible(a2, a3));
EXPECT_FALSE(softly_compatible(a3, a2));
}
test/unit/cute/core/packed_tuple.cpp
+581
View File
@@ -0,0 +1,581 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <cassert>
#include <cstdint>
#include <tuple>
#include <cute/container/tuple.hpp>
#include <cute/container/packed_tuple.hpp>
#include <cute/algorithm/tuple_algorithms.hpp>
#include <cute/tensor.hpp>
namespace pt_test {
template <class T>
struct Nonempty {
T datum;
Nonempty(T const& t) : datum{t} {}
friend bool operator==(Nonempty<T> const& lhs, Nonempty<T> const& rhs) {
return lhs.datum == rhs.datum;
}
friend bool operator!=(Nonempty<T> const& lhs, Nonempty<T> const& rhs) {
return !(lhs == rhs);
}
};
template <int V>
struct Empty {
template <int W>
friend bool operator==(Empty<V> const&, Empty<W> const&) {
return V == W;
}
template <int W>
friend bool operator!=(Empty<V> const& lhs, Empty<W> const& rhs) {
return !(lhs == rhs);
}
};
// std::tuple
static_assert(cute::is_standard_layout_v<std::tuple<>>); // it happens to be
static_assert(cute::is_standard_layout_v<std::tuple<int>>); // it happens to be
static_assert(cute::is_standard_layout_v<std::tuple<double>>); // it happens to be
static_assert(not cute::is_standard_layout_v<std::tuple<int, double>>); // it's not
#if ! defined(CUTLASS_USE_PACKED_TUPLE)
// cute::tuple
static_assert(cute::is_standard_layout_v<cute::tuple<>>); // it happens to be
static_assert(cute::is_standard_layout_v<cute::tuple<int>>); // it happens to be
static_assert(cute::is_standard_layout_v<cute::tuple<double>>); // it happens to be
static_assert(not cute::is_standard_layout_v<cute::tuple<int, double>>); // it's not
#endif // CUTLASS_USE_PACKED_TUPLE
// cute::packed_tuple
static_assert(cute::is_standard_layout_v<cute::packed_tuple<>>);
static_assert(cute::is_standard_layout_v<cute::packed_tuple<int>>);
static_assert(cute::is_standard_layout_v<cute::packed_tuple<double>>);
static_assert(cute::is_standard_layout_v<cute::packed_tuple<int, double>>); // it is
static_assert(cute::is_standard_layout_v<cute::packed_tuple<int, int, int, int>>); // it is
static_assert(cute::is_standard_layout_v<cute::packed_tuple<int, cute::packed_tuple<int, int>, int>>); // it is
static_assert(cute::is_standard_layout_v<cute::packed_tuple<int, cute::packed_tuple<Empty<0>, Empty<0>>, int>>); // it is
//////////////////////////////////////////////////////////////////////
// packed_tuple test starts here
//////////////////////////////////////////////////////////////////////
template <
class ExpectedPackedType,
size_t ExpectedPackedSize,
class ... Args>
constexpr void
test_packed_type_alias([[maybe_unused]] ExpectedPackedType packed, std::tuple<Args...> unpacked)
{
using cute::packed_tuple;
if constexpr ((cute::is_standard_layout_v<Args> && ...)) {
static_assert(cute::is_standard_layout_v<packed_tuple<Args...>>);
}
if constexpr ((cute::is_empty_v<Args> && ...)) {
static_assert(cute::is_empty_v<packed_tuple<Args...>>);
}
static_assert(cute::tuple_size_v<packed_tuple<Args...>> == sizeof...(Args));
auto test_element = [unpacked] (auto index) {
static_assert(cute::is_same_v<
std::tuple_element_t<index, packed_tuple<Args...>>,
std::tuple_element_t<index, std::tuple<Args...>>
>);
packed_tuple<Args...> sl = cute::apply(unpacked, [](auto... a){ return cute::make_packed_tuple(a...); });
EXPECT_EQ(std::get<index>(unpacked), cute::get<index>(sl));
};
cute::for_each(std::make_index_sequence<sizeof...(Args)>(), test_element);
}
void test_packed_type_aliases() {
using cute::packed_tuple;
test_packed_type_alias<packed_tuple<>, 0>({}, {});
test_packed_type_alias<packed_tuple<int>, 1, int>({7}, {7});
test_packed_type_alias<packed_tuple<double>, 1, double>({1.5}, {1.5});
// Make sure that class types are handled the same as scalar types
test_packed_type_alias<packed_tuple<Nonempty<int>>, 1, Nonempty<int>>(
{Nonempty{7}}, {Nonempty{7}});
test_packed_type_alias<packed_tuple<Nonempty<double>>, 1, Nonempty<double>>(
{Nonempty{1.5}}, {Nonempty{1.5}});
test_packed_type_alias<packed_tuple<>, 0, Empty<0>>({}, {});
test_packed_type_alias<packed_tuple<>, 0, Empty<0>, Empty<1>>(
{}, {Empty<0>{}, Empty<1>{}});
test_packed_type_alias<packed_tuple<>, 0, Empty<0>, Empty<1>, Empty<2>>(
{}, {Empty<0>{}, Empty<1>{}, Empty<2>{}});
test_packed_type_alias<packed_tuple<int>, 1, Empty<0>, int>(
{7}, {Empty<0>{}, 7});
test_packed_type_alias<packed_tuple<int>, 1, int, Empty<0>>(
{7}, {7, Empty<0>{}});
test_packed_type_alias<packed_tuple<int>, 1, int, Empty<0>, Empty<1>>(
{7}, {7, Empty<0>{}, Empty<1>{}});
test_packed_type_alias<packed_tuple<int>, 1, Empty<0>, int, Empty<1>>(
{7}, {Empty<0>{}, 7, Empty<1>{}});
test_packed_type_alias<packed_tuple<int>, 1, Empty<0>, Empty<1>, int>(
{7}, {Empty<0>{}, Empty<1>{}, 7});
test_packed_type_alias<packed_tuple<int, double>, 2, int, double, Empty<0>>(
{7, 1.5}, {7, 1.5, Empty<0>{}});
test_packed_type_alias<packed_tuple<int, double>, 2, int, Empty<0>, double>(
{7, 1.5}, {7, Empty<0>{}, 1.5});
test_packed_type_alias<packed_tuple<int, double>, 2, int, double, Empty<0>>(
{7, 1.5}, {7, 1.5, Empty<0>{}});
test_packed_type_alias<packed_tuple<int, double>, 2, int, double, Empty<0>, Empty<1>>(
{7, 1.5}, {7, 1.5, Empty<0>{}, Empty<1>{}});
test_packed_type_alias<packed_tuple<int, double>, 2, int, Empty<0>, double, Empty<1>>(
{7, 1.5}, {7, Empty<0>{}, 1.5, Empty<1>{}});
test_packed_type_alias<packed_tuple<int, double>, 2, int, Empty<0>, Empty<1>, double>(
{7, 1.5}, {7, Empty<0>{}, Empty<1>{}, 1.5});
test_packed_type_alias<packed_tuple<int, double>, 2, Empty<0>, int, Empty<1>, double>(
{7, 1.5}, {Empty<0>{}, 7, Empty<1>{}, 1.5});
test_packed_type_alias<packed_tuple<int, double>, 2, Empty<0>, Empty<1>, int, double>(
{7, 1.5}, {Empty<0>{}, Empty<1>{}, 7, 1.5});
test_packed_type_alias<packed_tuple<int, double, float>, 3, Empty<0>, int, double, float>(
{7, 1.5, 2.5f}, {Empty<0>{}, 7, 1.5, 2.5f});
test_packed_type_alias<packed_tuple<int, double, float>, 3, int, Empty<0>, double, float>(
{7, 1.5, 2.5f}, {7, Empty<0>{}, 1.5, 2.5f});
test_packed_type_alias<packed_tuple<int, double, float>, 3, int, double, Empty<0>, float>(
{7, 1.5, 2.5f}, {7, 1.5, Empty<0>{}, 2.5f});
test_packed_type_alias<packed_tuple<int, double, float>, 3, int, double, float, Empty<0>>(
{7, 1.5, 2.5f}, {7, 1.5, 2.5f, Empty<0>{}});
}
template <class Tuple, size_t Which, class ExpectedElementType>
constexpr bool test_tuple_element() {
return cute::is_same_v<std::tuple_element_t<Which, Tuple>, ExpectedElementType>;
}
void test_tuple_elements() {
using cute::packed_tuple;
static_assert(test_tuple_element<std::tuple<Empty<0>>, 0, Empty<0>>());
static_assert(test_tuple_element<packed_tuple<Empty<0>>, 0, Empty<0>>());
}
// A default-constructible type.
template <size_t Value>
struct DefaultConstructible {};
void test_default_constructibility() {
using cute::packed_tuple;
{
[[maybe_unused]] packed_tuple<> t_p_0;
[[maybe_unused]] packed_tuple<DefaultConstructible<0>> t_p_1;
[[maybe_unused]] packed_tuple<DefaultConstructible<0>, DefaultConstructible<1>> t_p_2;
[[maybe_unused]] packed_tuple<DefaultConstructible<0>, int, DefaultConstructible<1>> t_p_3;
}
}
void test_sizes_and_not_storing_empty_types() {
using cute::packed_tuple;
[[maybe_unused]] packed_tuple<
int,
pt_test::Empty<0>,
double
> pt{42, pt_test::Empty<0>{}, 1.5};
static_assert(cute::is_standard_layout_v<decltype(pt)>);
// packed_result_type must only store the packed tuple,
// and not the integer_sequence(s) used to access it.
// The latter can be represented entirely at compile time as types.
struct { int i; double j; } IntDouble;
static_assert(sizeof(pt) == sizeof(IntDouble));
EXPECT_EQ(cute::get<0>(pt), 42);
EXPECT_EQ(cute::get<1>(pt), pt_test::Empty<0>{});
EXPECT_EQ(cute::get<2>(pt), 1.5);
packed_tuple<
pt_test::Empty<0>,
pt_test::Empty<1>,
packed_tuple<
pt_test::Empty<0>,
pt_test::Empty<1>,
packed_tuple<pt_test::Empty<0>, packed_tuple<>>
>
> pt_empty{};
static_assert(cute::is_empty_v<decltype(pt_empty)>);
static_assert(cute::is_standard_layout_v<decltype(pt_empty)>);
static_assert(sizeof(pt_empty) == 1);
// Template arguments must be default constructible,
// and packed_tuple itself needs a default constructor.
[[maybe_unused]] packed_tuple<
packed_tuple<int, pt_test::Empty<2>>,
double,
pt_test::Empty<3>> pt2;
static_assert(cute::is_standard_layout_v<decltype(pt2)>);
// cute::packed_tuple, like the original cute::tuple, does not
// promise to have working CTAD (constructor template argument
// deduction).
[[maybe_unused]] packed_tuple<
packed_tuple<int, pt_test::Empty<0>>,
pt_test::Empty<1>
> pt3{
packed_tuple<int, pt_test::Empty<0>>{42, pt_test::Empty<0>{}},
pt_test::Empty<1>{}
};
static_assert(cute::is_standard_layout_v<decltype(pt3)>);
static_assert(cute::is_same_v<
cute::tuple_element_t<0, decltype(pt3)>,
packed_tuple<int, pt_test::Empty<0>>>);
static_assert(cute::is_same_v<
cute::tuple_element_t<1, decltype(pt3)>,
pt_test::Empty<1>>);
static_assert(cute::tuple_size_v<cute::tuple_element_t<0, decltype(pt3)>> == 2u);
packed_tuple<int, pt_test::Empty<0>> pt3_0 = cute::get<0>(pt3);
auto pt3_0_1 = cute::get<1>(pt3_0);
static_assert(cute::is_same_v<decltype(pt3_0_1), pt_test::Empty<0>>);
EXPECT_EQ(cute::get<0>(cute::get<0>(pt3)), 42);
EXPECT_EQ(cute::get<1>(cute::get<0>(pt3)), pt_test::Empty<0>{});
}
} // namespace test
TEST(CuTe_core, PackedTuple2)
{
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("packed_tuple");
CUTLASS_TRACE_HOST("-------------------------------");
pt_test::test_packed_type_aliases();
pt_test::test_tuple_elements();
pt_test::test_default_constructibility();
pt_test::test_sizes_and_not_storing_empty_types();
}
TEST(CuTe_core, PackedTuple2Get) {
using cute::packed_tuple;
using pt_test::Empty;
using pt_test::Nonempty;
{
using tuple_type = packed_tuple<int>;
tuple_type pt{42};
static_assert(cute::tuple_size_v<tuple_type> == 1u);
static_assert(cute::is_same_v<cute::tuple_element_t<0, tuple_type>, int>);
EXPECT_EQ(cute::get<0>(pt), 42);
cute::get<0>(pt) = 43;
EXPECT_EQ(cute::get<0>(pt), 43);
}
{
using tuple_type = packed_tuple<int>;
tuple_type const pt{42};
EXPECT_EQ(cute::get<0>(pt), 42);
static_assert(cute::is_same_v<decltype(cute::get<0>(pt)), int const&>);
}
{
EXPECT_EQ(cute::get<0>(packed_tuple<int>{42}), 42);
}
{
using tuple_type = packed_tuple<pt_test::Empty<0>>;
tuple_type pt;
static_assert(cute::tuple_size_v<tuple_type> == 1u);
static_assert(cute::is_same_v<cute::tuple_element_t<0, tuple_type>, pt_test::Empty<0>>);
EXPECT_EQ(cute::get<0>(pt), pt_test::Empty<0>{});
}
{
using tuple_type = packed_tuple<pt_test::Empty<0>>;
tuple_type const pt;
EXPECT_EQ(cute::get<0>(pt), pt_test::Empty<0>{});
}
{
using tuple_type = packed_tuple<pt_test::Empty<0>>;
EXPECT_EQ(cute::get<0>(tuple_type{}), pt_test::Empty<0>{});
}
{
using tuple_type = packed_tuple<int, double>;
tuple_type pt{1, 2.5};
static_assert(cute::tuple_size_v<tuple_type> == 2u);
static_assert(cute::is_same_v<cute::tuple_element_t<0, tuple_type>, int>);
static_assert(cute::is_same_v<cute::tuple_element_t<1, tuple_type>, double>);
EXPECT_EQ(cute::get<0>(pt), 1);
cute::get<0>(pt) = 2;
EXPECT_EQ(cute::get<0>(pt), 2);
EXPECT_EQ(cute::get<1>(pt), 2.5);
cute::get<1>(pt) = 3.5;
EXPECT_EQ(cute::get<1>(pt), 3.5);
}
{
using tuple_type = packed_tuple<int, double>;
tuple_type const pt{1, 2.5};
EXPECT_EQ(cute::get<0>(pt), 1);
static_assert(cute::is_same_v<decltype(cute::get<0>(pt)), int const&>);
EXPECT_EQ(cute::get<1>(pt), 2.5);
static_assert(cute::is_same_v<decltype(cute::get<1>(pt)), double const&>);
}
{
using tuple_type = packed_tuple<int, double>;
EXPECT_EQ(cute::get<0>(tuple_type{1, 2.5}), 1);
EXPECT_EQ(cute::get<1>(tuple_type{1, 2.5}), 2.5);
}
{
using tuple_type = packed_tuple<Empty<0>, double>;
tuple_type pt{Empty<0>{}, 2.5};
static_assert(cute::tuple_size_v<tuple_type> == 2u);
static_assert(cute::is_same_v<cute::tuple_element_t<0, tuple_type>, Empty<0>>);
static_assert(cute::is_same_v<cute::tuple_element_t<1, tuple_type>, double>);
EXPECT_EQ(cute::get<0>(pt), Empty<0>{});
EXPECT_EQ(cute::get<1>(pt), 2.5);
cute::get<1>(pt) = 3.5;
EXPECT_EQ(cute::get<1>(pt), 3.5);
}
{
using tuple_type = packed_tuple<Empty<0>, double>;
tuple_type const pt{Empty<0>{}, 2.5};
EXPECT_EQ(cute::get<0>(pt), Empty<0>{});
static_assert(cute::is_same_v<decltype(cute::get<0>(pt)), Empty<0>>);
EXPECT_EQ(cute::get<1>(pt), 2.5);
static_assert(cute::is_same_v<decltype(cute::get<1>(pt)), double const&>);
}
{
using tuple_type = packed_tuple<Empty<0>, double>;
EXPECT_EQ(cute::get<0>(tuple_type{Empty<0>{}, 2.5}), Empty<0>{});
EXPECT_EQ(cute::get<1>(tuple_type{Empty<0>{}, 2.5}), 2.5);
}
{
using tuple_type = packed_tuple<int, double, Nonempty<float>>;
tuple_type pt{1, 2.5, Nonempty{3.25f}};
static_assert(cute::tuple_size_v<tuple_type> == 3u);
static_assert(cute::is_same_v<cute::tuple_element_t<0, tuple_type>, int>);
static_assert(cute::is_same_v<cute::tuple_element_t<1, tuple_type>, double>);
static_assert(cute::is_same_v<cute::tuple_element_t<2, tuple_type>, Nonempty<float>>);
EXPECT_EQ(cute::get<0>(pt), 1);
EXPECT_EQ(cute::get<1>(pt), 2.5);
EXPECT_EQ(cute::get<2>(pt), Nonempty{3.25f});
cute::get<0>(pt) = 42;
EXPECT_EQ(cute::get<0>(pt), 42);
cute::get<1>(pt) = 4.5;
EXPECT_EQ(cute::get<1>(pt), 4.5);
cute::get<2>(pt) = Nonempty<float>{3.75f};
EXPECT_EQ(cute::get<2>(pt), Nonempty<float>{3.75f});
}
{
using tuple_type = packed_tuple<int, double, Nonempty<float>>;
tuple_type const pt{1, 2.5, Nonempty{3.25f}};
EXPECT_EQ(cute::get<0>(pt), 1);
EXPECT_EQ(cute::get<1>(pt), 2.5);
EXPECT_EQ(cute::get<2>(pt), Nonempty{3.25f});
}
{
using tuple_type = packed_tuple<int, double, Nonempty<float>>;
EXPECT_EQ((cute::get<0>(tuple_type{1, 2.5, Nonempty{3.25f}})), 1);
EXPECT_EQ((cute::get<1>(tuple_type{1, 2.5, Nonempty{3.25f}})), 2.5);
EXPECT_EQ((cute::get<2>(tuple_type{1, 2.5, Nonempty{3.25f}})), Nonempty{3.25f});
}
{
using tuple_type = packed_tuple<int, Empty<0>, Nonempty<float>>;
packed_tuple<int, Empty<0>, Nonempty<float>> pt{1, Empty<0>{}, Nonempty{3.25f}};
static_assert(cute::tuple_size_v<tuple_type> == 3u);
static_assert(cute::is_same_v<cute::tuple_element_t<0, tuple_type>, int>);
static_assert(cute::is_same_v<cute::tuple_element_t<1, tuple_type>, Empty<0>>);
static_assert(cute::is_same_v<cute::tuple_element_t<2, tuple_type>, Nonempty<float>>);
EXPECT_EQ(cute::get<0>(pt), 1);
EXPECT_EQ(cute::get<1>(pt), Empty<0>{});
EXPECT_EQ(cute::get<2>(pt), Nonempty{3.25f});
cute::get<0>(pt) = 42;
EXPECT_EQ(cute::get<0>(pt), 42);
cute::get<2>(pt) = Nonempty<float>{3.75f};
EXPECT_EQ(cute::get<2>(pt), Nonempty<float>{3.75f});
}
{
using tuple_type = packed_tuple<int, Empty<0>, Nonempty<float>>;
tuple_type const pt{1, Empty<0>{}, Nonempty{3.25f}};
EXPECT_EQ(cute::get<0>(pt), 1);
EXPECT_EQ(cute::get<1>(pt), Empty<0>{});
EXPECT_EQ(cute::get<2>(pt), Nonempty{3.25f});
}
{
using tuple_type = packed_tuple<int, Empty<0>, Nonempty<float>>;
EXPECT_EQ((cute::get<0>(tuple_type{1, Empty<0>{}, Nonempty{3.25f}})), 1);
EXPECT_EQ((cute::get<1>(tuple_type{1, Empty<0>{}, Nonempty{3.25f}})), Empty<0>{});
EXPECT_EQ((cute::get<2>(tuple_type{1, Empty<0>{}, Nonempty{3.25f}})), Nonempty{3.25f});
}
}
namespace pt_test {
// An empty class type to which Empty is convertible.
template<int Value>
struct ConvertibleFromEmpty {
constexpr ConvertibleFromEmpty() = default;
constexpr ConvertibleFromEmpty(Empty<Value>) {}
template <int OtherValue>
friend constexpr bool operator==(ConvertibleFromEmpty<Value> const&, ConvertibleFromEmpty<OtherValue> const&) {
return Value == OtherValue;
}
template <int OtherValue>
friend constexpr bool operator!=(ConvertibleFromEmpty<Value> const& lhs, ConvertibleFromEmpty<OtherValue> const& rhs) {
return !(lhs == rhs);
}
};
} // end namespace pt_test
TEST(CuTe_core, PackedTupleConstexprDefaultConstruction) {
// Make sure that packed_tuple's default constructor is constexpr.
// MSVC makes this a bit more challenging than usual.
using pt_test::Empty;
{
[[maybe_unused]] constexpr cute::detail::ESO_t<Empty<0>> eso1{};
[[maybe_unused]] constexpr cute::detail::ESO_t<int64_t> eso2{};
}
{
[[maybe_unused]] constexpr cute::detail::ESO_t<Empty<0>, Empty<1>> eso0{};
[[maybe_unused]] constexpr cute::detail::ESO_t<int64_t, Empty<1>> eso1{};
[[maybe_unused]] constexpr cute::detail::ESO_t<Empty<0>, int64_t> eso2{};
[[maybe_unused]] constexpr cute::detail::ESO_t<int64_t, int64_t> eso3{};
}
}
TEST(CuTe_core, PackedTupleConvertingConstruction) {
using cute::packed_tuple;
using pt_test::ConvertibleFromEmpty;
using pt_test::Empty;
using pt_test::Nonempty;
{
using tuple_type = cute::tuple<Nonempty<int>>;
[[maybe_unused]] tuple_type t(7);
EXPECT_EQ(cute::get<0>(t), Nonempty<int>(7));
}
{
using tuple_type = packed_tuple<Nonempty<int>>;
[[maybe_unused]] tuple_type t(7);
EXPECT_EQ(cute::get<0>(t), Nonempty<int>(7));
}
{
using tuple_type = cute::tuple<ConvertibleFromEmpty<0>>;
[[maybe_unused]] tuple_type t(Empty<0>{});
EXPECT_EQ(cute::get<0>(t), ConvertibleFromEmpty<0>{});
}
{
using tuple_type = packed_tuple<ConvertibleFromEmpty<0>>;
[[maybe_unused]] tuple_type t(Empty<0>{});
EXPECT_EQ(cute::get<0>(t), ConvertibleFromEmpty<0>{});
}
{
using tuple_type = cute::tuple<float, Nonempty<int>>;
[[maybe_unused]] tuple_type t(1.5f, 7);
EXPECT_EQ(cute::get<0>(t), 1.5f);
EXPECT_EQ(cute::get<1>(t), Nonempty<int>(7));
}
{
using tuple_type = packed_tuple<float, Nonempty<int>>;
[[maybe_unused]] tuple_type t(1.5f, 7);
EXPECT_EQ(cute::get<0>(t), 1.5f);
EXPECT_EQ(cute::get<1>(t), Nonempty<int>(7));
}
{
using tuple_type = cute::tuple<Empty<0>, Nonempty<int>>;
[[maybe_unused]] tuple_type t(Empty<0>{}, 7);
EXPECT_EQ(cute::get<0>(t), Empty<0>{});
EXPECT_EQ(cute::get<1>(t), Nonempty<int>(7));
}
{
using tuple_type = packed_tuple<Empty<0>, Nonempty<int>>;
[[maybe_unused]] tuple_type t(Empty<0>{}, 7);
EXPECT_EQ(cute::get<0>(t), Empty<0>{});
EXPECT_EQ(cute::get<1>(t), Nonempty<int>(7));
}
{
using tuple_type = cute::tuple<ConvertibleFromEmpty<0>, Nonempty<int>>;
[[maybe_unused]] tuple_type t(Empty<0>{}, 7);
EXPECT_EQ(cute::get<0>(t), ConvertibleFromEmpty<0>{});
EXPECT_EQ(cute::get<1>(t), Nonempty<int>(7));
}
{
using tuple_type = packed_tuple<ConvertibleFromEmpty<0>, Nonempty<int>>;
[[maybe_unused]] tuple_type t(Empty<0>{}, 7);
EXPECT_EQ(cute::get<0>(t), ConvertibleFromEmpty<0>{});
EXPECT_EQ(cute::get<1>(t), Nonempty<int>(7));
}
{
using inner_tuple_type = cute::tuple<Empty<0>>;
using outer_tuple_type = cute::tuple<inner_tuple_type>;
[[maybe_unused]] outer_tuple_type t(inner_tuple_type{Empty<0>{}});
}
{
using inner_tuple_type = packed_tuple<Empty<0>>;
using outer_tuple_type = packed_tuple<inner_tuple_type>;
[[maybe_unused]] outer_tuple_type t(inner_tuple_type{Empty<0>{}});
}
{
using inner_tuple_type = cute::tuple<ConvertibleFromEmpty<0>>;
using outer_tuple_type = cute::tuple<inner_tuple_type>;
[[maybe_unused]] outer_tuple_type t(inner_tuple_type{Empty<0>{}});
}
{
using inner_tuple_type = packed_tuple<ConvertibleFromEmpty<0>>;
using outer_tuple_type = packed_tuple<inner_tuple_type>;
[[maybe_unused]] outer_tuple_type t(inner_tuple_type{Empty<0>{}});
}
}
test/unit/cute/core/tuple_find.cpp
+103
View File
@@ -0,0 +1,103 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <cute/container/packed_tuple.hpp>
#include <cute/container/tuple.hpp>
namespace test {
template<size_t ExpectedIndex, class X, class Tuple>
void test_tuple_find(Tuple const& t) {
auto index = cute::find<X>(t);
static_assert(decltype(index)::value == ExpectedIndex);
}
template<template<class...> class Tuple>
void test_tuple_find_all() {
using test::test_tuple_find;
using cute::_1;
using cute::_2;
using cute::_4;
test_tuple_find<0, _1>(Tuple<_1>{});
test_tuple_find<0, int>(Tuple<int>{7});
test_tuple_find<0, _1>(Tuple<_1, _2>{});
test_tuple_find<0, _1>(Tuple<_1, int>{_1{}, 7});
test_tuple_find<0, float>(Tuple<float, int>{15.5f, 7});
test_tuple_find<1, _2>(Tuple<_1, _2>{});
test_tuple_find<1, int>(Tuple<_1, int>{_1{}, 7});
test_tuple_find<1, int>(Tuple<float, int>{15.5f, 7});
test_tuple_find<0, _1>(Tuple<_1, _2, _4>{_1{}, _2{}, _4{}});
test_tuple_find<0, _1>(Tuple<_1, _2, int>{_1{}, _2{}, 7});
test_tuple_find<0, _1>(Tuple<_1, float, _4>{_1{}, 15.5f, _4{}});
test_tuple_find<0, _1>(Tuple<_1, float, int>{_1{}, 15.5f, 7});
test_tuple_find<0, double>(Tuple<double, _2, _4>{105.5, _2{}, _4{}});
test_tuple_find<0, double>(Tuple<double, float, _4>{105.5, 15.5f, _4{}});
test_tuple_find<0, double>(Tuple<double, float, int>{105.5, 15.5f, 7});
test_tuple_find<1, _2>(Tuple<_1, _2, _4>{_1{}, _2{}, _4{}});
test_tuple_find<1, _2>(Tuple<_1, _2, int>{_1{}, _2{}, 7});
test_tuple_find<1, float>(Tuple<_1, float, _4>{_1{}, 15.5f, _4{}});
test_tuple_find<1, float>(Tuple<_1, float, int>{_1{}, 15.5f, 7});
test_tuple_find<1, _2>(Tuple<double, _2, _4>{105.5, _2{}, _4{}});
test_tuple_find<1, float>(Tuple<double, float, _4>{105.5, 15.5f, _4{}});
test_tuple_find<1, float>(Tuple<double, float, int>{105.5, 15.5f, 7});
test_tuple_find<2, _4>(Tuple<_1, _2, _4>{_1{}, _2{}, _4{}});
test_tuple_find<2, int>(Tuple<_1, _2, int>{_1{}, _2{}, 7});
test_tuple_find<2, _4>(Tuple<_1, float, _4>{_1{}, 15.5f, _4{}});
test_tuple_find<2, int>(Tuple<_1, float, int>{_1{}, 15.5f, 7});
test_tuple_find<2, _4>(Tuple<double, _2, _4>{105.5, _2{}, _4{}});
test_tuple_find<2, _4>(Tuple<double, float, _4>{105.5, 15.5f, _4{}});
test_tuple_find<2, int>(Tuple<double, float, int>{105.5, 15.5f, 7});
}
} // end namespace test
TEST(CuTe_core, TupleFind)
{
test::test_tuple_find_all<cute::tuple>();
}
// If cute::tuple is not simply an alias for cute::packed_tuple,
// then test cute::packed_tuple separately.
#if ! defined(CUTLASS_USE_PACKED_TUPLE)
TEST(CuTe_core, PackedTupleFind)
{
test::test_tuple_find_all<cute::packed_tuple>();
}
#endif // CUTLASS_USE_PACKED_TUPLE
test/unit/cute/hopper/CMakeLists.txt
+6
View File
@@ -29,6 +29,7 @@
add_custom_target(
cutlass_test_unit_cute_hopper
DEPENDS
cutlass_test_unit_cute_hopper_cooperative_gemm
cutlass_test_unit_cute_hopper_stsm
cutlass_test_unit_cute_hopper_tma_load
cutlass_test_unit_cute_hopper_tma_store
@@ -46,6 +47,11 @@ add_custom_target(
test_unit_cute_hopper_bulk_store
)
cutlass_test_unit_add_executable(
cutlass_test_unit_cute_hopper_cooperative_gemm
cooperative_gemm.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_cute_hopper_stsm
stsm.cu
test/unit/cute/hopper/cooperative_gemm.cu
+132
View File
@@ -0,0 +1,132 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include "../cooperative_gemm_common.hpp"
using namespace cute;
#define USE_FP8 1
#if USE_FP8
TEST(SM90_CuTe_Hopper, CooperativeGemmTilingF8) {
using TA = uint8_t;
using TB = uint8_t;
using TC = uint32_t;
constexpr uint32_t thread_block_size = 128;
constexpr int MaxVecBits = 16;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x32_S32S8S8S32_TN>,
Layout<Shape<_2, _2, _1>, Stride<_1, _2, _0>>,
Tile<_32, _32, _32>
>;
using swizzle = Swizzle<2, 4, 3>;
// This is for A row major, B col major according to CUTLASS default configs
using ALayout = decltype(composition(swizzle{}, Layout<Shape<_64, _64>, Stride<_64, _1>>{}));
using BLayout = decltype(composition(swizzle{}, Layout<Shape<_64, _64>, Stride<_1, _64>>{}));
using CLayout = decltype(make_layout(Shape<_64, _64>{}, LayoutLeft{}));
test_cooperative_gemm<ALayout,
BLayout,
CLayout,
ALayout,
BLayout,
CLayout,
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // A
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // B
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // C
thread_block_size,
tiled_mma_t,
MaxVecBits,
TA,
TB,
TC>();
}
#else
TEST(SM90_CuTe_Hopper, CooperativeGemmTilingF16) {
using TA = half_t;
using TB = half_t;
using TC = half_t;
constexpr uint32_t thread_block_size = 64;
constexpr int MaxVecBits = 16;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x16_F16F16F16F16_TN>,
Layout<Shape<_2, _1, _1>, Stride<_1, _0, _0>>,
Tile<_32, _32, _32>
>;
using swizzle = Swizzle<3, 3, 3>;
// This is for A row major, B col major according to CUTLASS default configs
using ALayout = decltype(composition(swizzle{},
Layout<Shape<_64, _64>, Stride<_64, _1>>{}));
using BLayout = decltype(composition(swizzle{},
Layout<Shape<_64, _64>, Stride<_1, _64>>{}));
using CLayout = decltype(make_layout(Shape<_64, _64>{}, LayoutLeft{}));
test_cooperative_gemm<ALayout,
BLayout,
CLayout,
ALayout,
BLayout,
CLayout,
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // A
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // B
AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>, // C
thread_block_size,
tiled_mma_t,
MaxVecBits,
TA,
TB,
TC>();
}
#endif
test/unit/cute/hopper/tma_load_testbed.hpp
+5
View File
@@ -122,6 +122,11 @@ tma_test_device_cute(T const* g_in, T* g_out,
}
#endif
// Test L2 prefetch
if (threadIdx.x == 0) {
prefetch(tma, tAgA);
}
// Loop over the TMA stages, using smem as our buffer
for (int stage = 0; stage < size<1>(tAgA); ++stage)
{
test/unit/cute/hopper/tma_store_testbed.hpp
+3
View File
@@ -117,6 +117,9 @@ tma_test_device_cute(T const* g_in, T* g_out,
}
#endif
// Test L2 prefetch
cooperative_prefetch<128>(threadIdx.x, gA);
// Loop over the TMA stages, using smem as our buffer
for (int stage = 0; stage < size<1>(tBgB); ++stage)
{
test/unit/cute/msvc_compilation/tuple.cpp
+3 -8
View File
@@ -53,6 +53,8 @@ private:
template<class Integral, Integral Value>
using IC = std::integral_constant<Integral, Value>;
#if ! defined(CUTLASS_USE_PACKED_TUPLE)
TEST(CuTe_core_msvc_compilation, TupleAssignment)
{
CUTLASS_TRACE_HOST("-------------------------------");
@@ -89,29 +91,22 @@ TEST(CuTe_core_msvc_compilation, TupleAssignment)
using tuple_0d_type = cute::tuple<>;
using tuple_1d_d_type = cute::tuple<int>;
using tuple_1d_s_type = cute::tuple<forty_two_type>;
using tuple_2d_dd_type = cute::tuple<int, size_t>;
using tuple_2d_ss_type = cute::tuple<forty_two_type, forty_three_type>;
[[maybe_unused]] tuple_0d_type t0;
// Symptom: "illegal member initialization: 'TupleBase<int>' is not a base or member"
[[maybe_unused]] tuple_1d_d_type t1{ 42 };
[[maybe_unused]] tuple_1d_s_type t2;
[[maybe_unused]] tuple_1d_d_type t1a{ 43 };
t1 = t1a;
[[maybe_unused]] tuple_2d_dd_type t3{ 42, size_t(43u) };
[[maybe_unused]] tuple_2d_ss_type t4;
t3 = t4;
[[maybe_unused]] tuple_2d_dd_type t3a{ 44, size_t(45u) };
// Symptom: "illegal member initialization:
// 'TupleBase<int, unsigned __int64>' is not a base or member"
t3 = t3a;
}
#endif // CUTLASS_USE_PACKED_TUPLE
TEST(CuTe_core_msvc_compilation, TupleGetSingleInteger)
{
test/unit/cute/volta/CMakeLists.txt
-1
View File
@@ -29,6 +29,5 @@
cutlass_test_unit_add_executable(
cutlass_test_unit_cute_volta
vectorization_auto.cu
cooperative_copy.cu
cooperative_gemm.cu
)
test/unit/cute/volta/cooperative_copy.cu
-486
View File
@@ -1,486 +0,0 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <iostream>
#include <iomanip>
#include <utility>
#include <type_traits>
#include <vector>
#include <numeric>
#include <tuple>
#include <thrust/host_vector.h>
#include <thrust/device_vector.h>
#include <cute/tensor.hpp>
#include <cute/numeric/numeric_types.hpp>
using namespace cute;
namespace cooperative_copy_mode {
struct global_shared {};
struct global_global {};
struct shared_shared {};
}
// gs --> global to/from shared
template <int MaxVecBits, class GMemLayout, class SMemLayout, uint32_t ThreadBlockSize, class T>
__device__ void
cooperative_copy_default_gs(T const* g_in, T* g_out)
{
using namespace cute;
extern __shared__ float4 smem_buf[];
// Cast smem_buf to smem_uint8_ptr and move it by MaxVecBits bits
// This is to make sure tests pass on pointer aligned to MaxVecBits bits
uint8_t* smem_uint8_ptr = reinterpret_cast<uint8_t*>(smem_buf) + (MaxVecBits/8);
T* smem = reinterpret_cast<T*>(smem_uint8_ptr);
Tensor g_in_tensor = make_tensor(make_gmem_ptr(g_in), GMemLayout{});
Tensor g_out_tensor = make_tensor(make_gmem_ptr(g_out), GMemLayout{});
Tensor s_tensor = make_tensor(make_smem_ptr(smem), SMemLayout{});
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, g_in_tensor, s_tensor);
__syncthreads();
if(thread0()) {
for(int i = 0; i < size(s_tensor); ++i) {
s_tensor(i) += T(i);
}
}
__syncthreads();
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, s_tensor, g_out_tensor);
}
// ss --> shared to shared
template <int MaxVecBits, class Layout1, class Layout2, uint32_t ThreadBlockSize, class T>
__device__ void
cooperative_copy_default_ss(T const* g_in, T* g_out)
{
using namespace cute;
extern __shared__ float4 smem_buf[];
// Cast smem_buf to smem_uint8_ptr and move it by MaxVecBits bits
// This is to make sure tests pass on pointer aligned to MaxVecBits bits
T* smem1 = reinterpret_cast<T*>(smem_buf);
uint8_t* smem2_uint8_ptr = reinterpret_cast<uint8_t*>(smem_buf) + (MaxVecBits/8);
T* smem2 = reinterpret_cast<T*>(smem2_uint8_ptr) + cute::cosize(Layout2{});
Tensor g_in_tensor = make_tensor(make_gmem_ptr(g_in), Layout1 {});
Tensor g_out_tensor = make_tensor(make_gmem_ptr(g_out), Layout2 {});
Tensor s1_tensor = make_tensor(make_smem_ptr(smem1), Layout2 {});
Tensor s2_tensor = make_tensor(make_smem_ptr(smem2), Layout1 {});
cooperative_copy<ThreadBlockSize, cute::sizeof_bits_v<T>>(threadIdx.x, g_in_tensor, s1_tensor);
__syncthreads();
if(thread0()) {
for(int i = 0; i < size(s1_tensor); ++i) {
s1_tensor(i) += T(i);
}
}
__syncthreads();
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, s1_tensor, s2_tensor);
__syncthreads();
cooperative_copy<ThreadBlockSize, cute::sizeof_bits_v<T>>(threadIdx.x, s2_tensor, g_out_tensor);
}
// gg --> global to global
template <int MaxVecBits, class Layout1, class Layout2, uint32_t ThreadBlockSize, class T>
__device__ void
cooperative_copy_default_gg(T const* g_in, T* g_out)
{
using namespace cute;
Tensor g_in_tensor = make_tensor(make_gmem_ptr(g_in), Layout1{});
Tensor g_out_tensor = make_tensor(make_gmem_ptr(g_out), Layout2{});
cooperative_copy<ThreadBlockSize, MaxVecBits>(threadIdx.x, g_in_tensor, g_out_tensor);
}
template <class Mode, int MaxVecBits, class Layout1, class Layout2, uint32_t ThreadBlockSize, class T>
__global__ void
cooperative_copy_default_kernel(T const* g_in, T* g_out)
{
if constexpr(std::is_same_v<Mode, cooperative_copy_mode::global_shared>) {
cooperative_copy_default_gs<MaxVecBits, Layout1, Layout2, ThreadBlockSize>(g_in, g_out);
} else if constexpr (std::is_same_v<Mode, cooperative_copy_mode::global_global>) {
cooperative_copy_default_gg<MaxVecBits, Layout1, Layout2, ThreadBlockSize>(g_in, g_out);
} else if constexpr (std::is_same_v<Mode, cooperative_copy_mode::shared_shared>) {
cooperative_copy_default_ss<MaxVecBits, Layout1, Layout2, ThreadBlockSize>(g_in, g_out);
}
}
// Mode - defines memory types of src and dst in cooperative_copy operation
// MaxVecBits - defines max vectorization in cooperative_copy operation, and enforces that
// alignment on used pointers to ensure correct testing
template <class Mode, int MaxVecBits, class Layout1, class Layout2, uint32_t ThreadBlockSize, class T>
void test_cooperative_copy_default()
{
using value_type = T;
static_assert(cute::size(Layout1{}) == cute::size(Layout2{}));
using gmem_layout_in = Layout1;
using gmem_layout_out = std::conditional_t<std::is_same_v<Mode, cooperative_copy_mode::global_shared>, Layout1, Layout2>;
#if 0
print(" "); print("layout1: "); print(Layout1{}); print("\n");
print(" "); print("layout2: "); print(Layout2{}); print("\n");
print(" "); print("threads: "); print(ThreadBlockSize); print("\n");
#endif
if constexpr (MaxVecBits < cute::sizeof_bits_v<value_type>) {
GTEST_SKIP() << "Skipping test since MaxVecBits (=" << MaxVecBits
<< ") < cute::sizeof_bits_v<value_type> (=" << cute::sizeof_bits_v<value_type> << ")";
} else {
constexpr auto max_vec_bytes = MaxVecBits / 8;
static_assert((max_vec_bytes % sizeof(T)) == 0);
constexpr uint32_t count = cute::cosize(gmem_layout_in {});
// Extra elements to force MaxVecBits alignment in global memory
constexpr uint32_t extra_elements = max_vec_bytes / sizeof(value_type);
// Allocate
thrust::host_vector<value_type> h_in(count + extra_elements);
thrust::host_vector<value_type> h_out(count + extra_elements);
// Initialize
Tensor h_in_tensor = make_tensor((h_in.data() + extra_elements), gmem_layout_in {});
Tensor h_out_tensor = make_tensor((h_out.data() + extra_elements), gmem_layout_out {});
for (int i = 0; i < cute::size(h_in_tensor); ++i) {
h_in_tensor(i) = value_type(float(i));
// For global-to-global copy need to compare against the same value
h_out_tensor(i) = std::is_same_v<Mode, cooperative_copy_mode::global_global> ? value_type(float(i)) : value_type(float(2 * i));
}
// To GPU
thrust::device_vector<value_type> d_in = h_in;
thrust::device_vector<value_type> d_out(d_in.size(), value_type(float(-2)));
// Adds (MaxVecBits/8) bytes to shared memory as we'll move pointer by that many bytes inside the kernel to enforce
// alignment to (MaxVecBits/8) bytes
size_t shared_memory_bytes = (sizeof(value_type) * count) + max_vec_bytes;
shared_memory_bytes += std::is_same_v<Mode, cooperative_copy_mode::shared_shared> * (sizeof(value_type) * count);
// Launch
auto coop_copy = cooperative_copy_default_kernel<Mode, MaxVecBits, Layout1, Layout2, ThreadBlockSize, value_type>;
ASSERT_EQ(cudaFuncSetAttribute(coop_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(shared_memory_bytes)), cudaSuccess);
auto d_in_ptr = thrust::raw_pointer_cast(d_in.data() + extra_elements);
auto d_out_ptr = thrust::raw_pointer_cast(d_out.data() + extra_elements);
coop_copy<<<1, ThreadBlockSize, shared_memory_bytes>>>(d_in_ptr, d_out_ptr);
cudaError_t result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
cudaError_t error = cudaGetLastError();
FAIL() << "Error at kernel sync: " << cudaGetErrorString(error) << "\n";
}
// Validate
thrust::host_vector<value_type> h_result = d_out;
Tensor h_result_tensor = make_tensor((h_result.data() + extra_elements), gmem_layout_out {});
for (int i = 0; i < cute::size(h_in_tensor); ++i) {
ASSERT_EQ(h_result_tensor(i), h_out_tensor(i))
<< i << " - result:" << h_result_tensor(i) << " expected:" << h_out_tensor(i);
}
}
}
template<class T>
class SM70_CuTe_Volta;
template<class Mode, class MaxVecBits>
class SM70_CuTe_Volta<std::tuple<Mode, MaxVecBits>>: public testing::Test
{
public:
using mode = Mode;
static constexpr int max_vec_bits = MaxVecBits::value;
};
typedef testing::Types<
std::tuple<cooperative_copy_mode::global_shared, cute::Int<128>>,
std::tuple<cooperative_copy_mode::global_shared, cute::Int<64>>,
std::tuple<cooperative_copy_mode::global_shared, cute::Int<32>>,
std::tuple<cooperative_copy_mode::global_shared, cute::Int<16>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<128>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<64>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<32>>,
std::tuple<cooperative_copy_mode::global_global, cute::Int<16>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<128>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<64>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<32>>,
std::tuple<cooperative_copy_mode::shared_shared, cute::Int<16>>,
> CooperativeCopyModeMaxVecBitsList;
TYPED_TEST_SUITE(SM70_CuTe_Volta, CooperativeCopyModeMaxVecBitsList);
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefault1D)
{
using value_type = float;
constexpr uint32_t count = 512;
using gmem_layout_t = decltype(make_layout(make_shape(Int<count>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<count>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefault1DFallback)
{
using value_type = float;
constexpr uint32_t count = 99;
using gmem_layout_t = decltype(make_layout(make_shape(Int<count>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<count>{})));
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2D)
{
using value_type = float;
constexpr uint32_t x = 32;
constexpr uint32_t y = 32;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2DFallback)
{
using value_type = float;
constexpr uint32_t x = 37;
constexpr uint32_t y = 37;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2DCustomStride)
{
using value_type = float;
constexpr uint32_t x = 16;
constexpr uint32_t y = 16;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(Int<y>{}, Int<1>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}), make_stride(Int<1>{}, Int<x>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG3D)
{
using value_type = cute::half_t;
constexpr uint32_t x = 8;
constexpr uint32_t y = 8;
constexpr uint32_t z = 16;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG3DFallback)
{
using value_type = cute::half_t;
constexpr uint32_t x = 44;
constexpr uint32_t y = 24;
constexpr uint32_t z = 14;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{})));
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2Dto3D)
{
using value_type = double;
constexpr uint32_t x = 16;
constexpr uint32_t y = 16;
constexpr uint32_t z = 4;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y*z>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<z>{}, Int<y>{}, Int<x>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSGCustom1)
{
using value_type = double;
using gmem_layout_t = decltype(make_layout(
make_shape(Int<8>{}, make_shape(Int<2>{}, Int<2>{})),
make_stride(Int<2>{}, make_shape(Int<1>{}, Int<16>{}))
));
using smem_layout_t = decltype(make_layout(
make_shape(Int<8>{}, Int<4>{}),
make_stride(Int<4>{}, Int<1>{})
));
constexpr uint32_t thread_block_size = 8;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSGCustom2)
{
using value_type = float;
using gmem_layout_t = decltype(make_layout(
make_shape(make_shape(Int<4>{}, Int<2>{}), make_shape(Int<2>{}, Int<2>{})),
make_stride(make_shape(Int<4>{}, Int<1>{}), make_shape(Int<16>{}, Int<2>{}))
));
using smem_layout_t = decltype(make_layout(
make_shape(make_shape(Int<2>{}, Int<2>{}, Int<2>{}), make_shape(Int<2>{}, Int<2>{})),
make_stride(make_shape(Int<16>{}, Int<4>{}, Int<1>{}), make_shape(Int<8>{}, Int<2>{}))
));
constexpr uint32_t thread_block_size = 16;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSGSwizzle1)
{
using value_type = float;
using gmem_layout_t = Layout<Shape<_8, _64>, Stride<_64, _1>>;
using smem_layout_t = decltype(composition(Swizzle<3, 3, 3>{}, Layout<Shape<_8, _64>, Stride<_64, _1>>{}));
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSGSwizzle2)
{
using value_type = cute::half_t;
using gmem_layout_t = decltype(make_layout(make_shape(Int<64>{}, Int<64>{})));
using smem_atom_layout_t = decltype(composition(Swizzle<3, 2, 3> {}, Layout<Shape<_8, _32>, Stride<_32, _1>>{}));
using smem_layout_t = decltype(tile_to_shape(
smem_atom_layout_t{},
make_shape(shape<0>(gmem_layout_t{}), shape<1>(gmem_layout_t{})))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSGSwizzle3)
{
using value_type = cute::half_t;
using gmem_layout_t = decltype(make_layout(make_shape(Int<64>{}, Int<64>{})));
using smem_atom_layout_t = decltype(composition(Swizzle<2, 4, 3> {}, Layout<Shape<_16, _64>, Stride<_64, _1>>{}));
using smem_layout_t = decltype(tile_to_shape(
smem_atom_layout_t{},
make_shape(shape<0>(gmem_layout_t{}), shape<1>(gmem_layout_t{})))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSGSwizzle4)
{
using value_type = cute::half_t;
using gmem_atom_layout_t = decltype(composition(Swizzle<3, 2, 3> {}, Layout<Shape<_8, _32>, Stride<_32, _1>>{}));
using smem_layout_t = decltype(make_layout(make_shape(Int<64>{}, Int<64>{})));
using gmem_layout_t = decltype(tile_to_shape(
gmem_atom_layout_t{},
make_shape(shape<0>(smem_layout_t{}), shape<1>(smem_layout_t{})))
);
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
test/unit/epilogue/warp/fragment_iterator_tensor_op.cu
+81 -5
View File
@@ -78,7 +78,7 @@ TEST(SM75_Epilogue_warp_FragmentIterator, mma_f32_64x64x8) {
std::cout << "Native accumulators:\n";
for (int i = 0; i < MmaTensorOp::FragmentC::kElements; ++i) {
for (size_t i = 0; i < MmaTensorOp::FragmentC::kElements; ++i) {
accum[i] = ElementC(i);
std::cout << accum[i] << " ";
@@ -106,7 +106,7 @@ TEST(SM75_Epilogue_warp_FragmentIterator, mma_f32_64x64x8) {
std::cout << "Iteration " << iter << ":\n";
for (int i = 0; i < FragmentIterator::Fragment::kElements; ++i) {
for (size_t i = 0; i < FragmentIterator::Fragment::kElements; ++i) {
std::cout << frag[i] << " ";
}
@@ -153,8 +153,8 @@ TEST(SM75_Epilogue_warp_FragmentIterator, mma_f16_64x64x8) {
std::cout << "Native accumulators:\n";
for (int i = 0; i < MmaTensorOp::FragmentC::kElements; ++i) {
accum[i] = ElementC(i);
for (size_t i = 0; i < MmaTensorOp::FragmentC::kElements; ++i) {
accum[i] = ElementC((int)i);
std::cout << (float)accum[i] << " ";
if (i && !((i + 1) % 4)) {
@@ -181,7 +181,7 @@ TEST(SM75_Epilogue_warp_FragmentIterator, mma_f16_64x64x8) {
std::cout << "Iteration " << iter << ":\n";
for (int i = 0; i < FragmentIterator::Fragment::kElements; ++i) {
for (size_t i = 0; i < FragmentIterator::Fragment::kElements; ++i) {
std::cout << (float)frag[i] << " ";
}
@@ -191,4 +191,80 @@ TEST(SM75_Epilogue_warp_FragmentIterator, mma_f16_64x64x8) {
}
#endif
}
TEST(SM75_Epilogue_warp_FragmentIterator_column, mma_f32_64x64x8) {
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
using Element = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 64>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
Shape,
InstructionShape,
Element,
LayoutA,
Element,
LayoutB,
ElementC,
cutlass::layout::RowMajor
>::Type;
using FragmentIterator = cutlass::epilogue::warp::FragmentIteratorTensorOp<
Shape,
typename MmaTensorOp::Policy::Operator::Shape,
typename MmaTensorOp::Policy::Operator::ElementC,
typename MmaTensorOp::Policy::Operator::FragmentC,
cutlass::layout::ColumnMajor
>;
// This test just prints things.
#if 0
typename MmaTensorOp::FragmentC accum;
std::cout << "Native accumulators:\n";
for (size_t i = 0; i < MmaTensorOp::FragmentC::kElements; ++i) {
accum[i] = ElementC((int)i);
std::cout << (float)accum[i] << " ";
if (i && !((i + 1) % 4)) {
std::cout << "\n";
}
}
std::cout << std::endl;
std::cout << "FragmentIterator::Policy = { \n"
<< " kAccessesPerInstruction: " << FragmentIterator::Policy::kIterationsPerInstruction << "\n"
<< " kAccumulatorRowStride: " << FragmentIterator::Policy::kAccumulatorRowStride << "\n"
<< " kAccumulatorColumnStride: " << FragmentIterator::Policy::kAccumulatorColumnStride << "\n"
<< " kIterations: " << FragmentIterator::Policy::kIterations << "\n"
<< " }" << std::endl;
FragmentIterator fragment_iterator(accum);
for (int iter = 0; iter < FragmentIterator::kIterations; ++iter) {
typename FragmentIterator::Fragment frag;
fragment_iterator.load(frag);
std::cout << "Iteration " << iter << ":\n";
for (size_t i = 0; i < FragmentIterator::Fragment::kElements; ++i) {
std::cout << (float)frag[i] << " ";
}
std::cout << std::endl;
++fragment_iterator;
}
#endif
}
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/CMakeLists.txt
+13 -2
View File
@@ -310,6 +310,18 @@ cutlass_test_unit_add_executable(
sm90_gemm_tf32_tf32_f32_alignx_tensor_op_f32_warpspecialized_pingpong.cu
)
# Ptr Array test
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_sm90_ptr_array
sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array.cu
)
# Group Gemm test
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_sm90_group_gemm
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
)
# Fused epilogue tests
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_epilogue_fusion_sm90
@@ -348,7 +360,6 @@ cutlass_test_unit_add_executable(
sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative.cu
sm90_gemm_f8_f8_f32_tensor_op_f32_cluster_warpspecialized_cooperative.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_gmma_rs_warpspecialized_sm90
@@ -508,7 +519,7 @@ cutlass_test_unit_add_executable(
gemm_f8t_f8n_f32t_tensor_op_f32_sm89.cu
gemm_f8t_f8n_f32t_tensor_op_f32_sparse_sm89.cu
gemm_f8t_f8n_f8t_tensor_op_f32_sm89.cu
gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
# gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
)
cutlass_test_unit_add_executable(
test/unit/gemm/device/gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32_sm80.cu
+2 -2
View File
@@ -162,7 +162,7 @@ TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 64x64x16_32x32x16) {
TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
@@ -194,7 +194,7 @@ TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16)
TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 64x128x16_32x64x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
test/unit/gemm/device/gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32_sm80.cu
+3 -3
View File
@@ -161,7 +161,7 @@ TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 64x64x16_32x32x16) {
TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
@@ -193,7 +193,7 @@ TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16)
TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 64x128x16_32x64x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
@@ -225,7 +225,7 @@ TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 64x128x16_32x64x16)
TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 128x128x16_32x64x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
test/unit/gemm/device/gemm_f16n_f16n_f16n_direct_store_tensor_op_f32_sm80.cu
+1 -1
View File
@@ -79,7 +79,7 @@ TEST(SM80_Device_GemmUniversal_DirectStore_f16n_f16t_f32n_tensor_op_f32, 128x128
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
4, // This is the vector size of the epilogue.
4,
ElementAccumulator,
ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
test/unit/gemm/device/gemm_f16n_f16n_f16t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -91,7 +91,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -109,7 +109,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -127,7 +127,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -145,7 +145,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -163,7 +163,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -199,7 +199,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -217,7 +217,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -235,7 +235,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -253,7 +253,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f16n_f16n_f32t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -73,7 +73,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -91,7 +91,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -109,7 +109,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -127,7 +127,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -145,7 +145,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -164,7 +164,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -182,7 +182,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -200,7 +200,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -218,7 +218,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -236,7 +236,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -254,7 +254,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f16_sparse_sm80.cu
+14 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x256x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 256x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 256x64x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x256x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x128x64_32x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -163,7 +163,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x64x64_64x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x64x64_32x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -199,7 +199,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x128x128_64x64x128
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -217,7 +217,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 256x64x128_64x64x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -235,7 +235,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x64x128_64x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -253,7 +253,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x64x128_32x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -266,6 +266,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x64x128_32x32x128)
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_f16n_f16t_f32t_tensor_op_f32_sparse_sm80.cu
+212 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -162,7 +162,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -180,7 +180,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -198,7 +198,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -216,7 +216,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -234,7 +234,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -252,7 +252,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -266,6 +266,205 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x64x128_32x32x128)
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x32x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 64>,
cutlass::gemm::GemmShape<32, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x32x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x32x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 64>,
cutlass::gemm::GemmShape<64, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x32x128_64x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 128>,
cutlass::gemm::GemmShape<64, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 32x128x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 32x256x128_32x64x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 128>,
cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x16x64_32x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 64>,
cutlass::gemm::GemmShape<32, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x16x128_32x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 128>,
cutlass::gemm::GemmShape<32, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x16x64_64x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 64>,
cutlass::gemm::GemmShape<64, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x16x128_64x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 128>,
cutlass::gemm::GemmShape<64, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED
test/unit/gemm/device/gemm_f16t_f16n_f16t_tensor_op_f16_sparse_sm80.cu
+13 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x256x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 256x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -109,7 +109,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 256x64x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -127,7 +127,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x256x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -145,7 +145,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x128x64_32x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -163,7 +163,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x64x64_64x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x64x64_32x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -199,7 +199,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x128x128_64x64x128
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -218,7 +218,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 256x64x128_64x64x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -236,7 +236,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x64x128_64x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -254,7 +254,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x64x128_32x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f16t_f16n_f32t_tensor_op_f32_sparse_sm80.cu
+213 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -162,7 +162,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -180,7 +180,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -198,7 +198,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -216,7 +216,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -234,7 +234,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -252,7 +252,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -266,6 +266,206 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x64x128_32x32x128)
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 32x128x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 32x256x128_32x64x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 128>,
cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x32x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 64>,
cutlass::gemm::GemmShape<32, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x32x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x32x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 64>,
cutlass::gemm::GemmShape<64, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x32x128_64x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 128>,
cutlass::gemm::GemmShape<64, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x16x64_32x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 64>,
cutlass::gemm::GemmShape<32, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x16x128_32x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 128>,
cutlass::gemm::GemmShape<32, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x16x64_64x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 64>,
cutlass::gemm::GemmShape<64, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x16x128_64x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 128>,
cutlass::gemm::GemmShape<64, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED
test/unit/gemm/device/gemm_f16t_f16t_f32t_tensor_op_f32_sparse_sm80.cu
+9 -9
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -162,7 +162,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -180,7 +180,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f32n_f32n_f32t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
test/unit/gemm/device/gemm_f32n_f32t_f32t_tensor_op_f32_sparse_sm80.cu
+107 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -424,6 +424,100 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 32x128x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 32x256x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 32>,
cutlass::gemm::GemmShape<32, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_f32t_f32n_f32t_tensor_op_f32_sparse_sm80.cu
+357 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -423,6 +423,350 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 32x128x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 32x256x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 32>,
cutlass::gemm::GemmShape<32, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x32x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x32x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x32x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 32>,
cutlass::gemm::GemmShape<64, 32, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x32x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x16x32_32x16x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 32>,
cutlass::gemm::GemmShape<32, 16, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x16x64_32x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 64>,
cutlass::gemm::GemmShape<32, 16, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x16x32_64x16x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 32>,
cutlass::gemm::GemmShape<64, 16, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x16x64_64x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 64>,
cutlass::gemm::GemmShape<64, 16, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_f32t_f32t_f32t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
test/unit/gemm/device/gemm_f8t_f8n_f32t_tensor_op_f32_sparse_sm89.cu
+5 -5
View File
@@ -41,7 +41,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -65,7 +65,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -89,7 +89,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe5m2n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -113,7 +113,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe4m3n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -137,7 +137,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe5m2n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
test/unit/gemm/device/gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
+47 -13
View File
@@ -43,7 +43,7 @@
#include "cutlass/cutlass.h"
#include "cutlass/epilogue/thread/activation.h"
#include "cutlass/epilogue/thread/linear_combination_generic_with_scaling.h"
#include "cutlass/gemm/device/gemm_sparse_with_absmax.h"
#include "cutlass/gemm/device/gemm_sparse_universal_with_absmax.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -78,7 +78,41 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
EpilogueOutputOp, cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, kStages
>;
bool passed = test::gemm::device::TestAllGemmWithAbsmax<Gemm, test::gemm::device::SparseTestbed<Gemm>, cutlass::epilogue::thread::Identity>();
EXPECT_TRUE(passed);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_f32t_tensor_op_f32, identity_128x128x128_64x64x128) {
// Test with float D and Aux for testing split-K without needing relative equality checks
using ElementA = cutlass::float_e4m3_t;
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = float;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
cutlass::epilogue::thread::Identity,
ElementOutput,
ElementAuxOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -112,12 +146,12 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_fastac
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
EpilogueOutputOp, cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, kStages,
kAlignment, kAlignment, false, cutlass::arch::OpMultiplyAddFastAccum
kAlignment, kAlignment, cutlass::arch::OpMultiplyAddFastAccum
>;
bool passed = test::gemm::device::TestAllGemmWithAbsmax<Gemm, test::gemm::device::SparseTestbed<Gemm>, cutlass::epilogue::thread::Identity>();
@@ -146,7 +180,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, relu_128x128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -179,7 +213,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe5m2n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -212,7 +246,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe4m3n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -245,7 +279,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe5m2n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -278,7 +312,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe5m2t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -311,7 +345,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe5m2n_fe5m2t_tensor_op_f32, identity_diff_a
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -344,7 +378,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_128x64
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 64, 128>, cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -377,7 +411,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_noScal
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -414,7 +448,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_noAux_
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
test/unit/gemm/device/gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
+278 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -55,7 +55,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x256x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -74,7 +74,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x128x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -93,7 +93,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x128x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -113,7 +113,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x64x256_64x64x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -132,7 +132,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x256x256_64x64x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -151,7 +151,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x128x256_32x64x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -170,7 +170,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x64x256_64x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -189,7 +189,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x256_32x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -208,7 +208,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x128x512_64x64x512)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -228,7 +228,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x64x512_64x32x512) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -247,7 +247,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x512_32x32x512) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -261,7 +261,272 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x512_32x32x512) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x128x512_32x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 512>,
cutlass::gemm::GemmShape<32, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x128x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x256x256_32x64x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 256>,
cutlass::gemm::GemmShape<32, 64, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x256x512_32x64x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 512>,
cutlass::gemm::GemmShape<32, 64, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 16x128x512_16x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 128, 512>,
cutlass::gemm::GemmShape<16, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 16x256x512_16x64x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 256, 512>,
cutlass::gemm::GemmShape<16, 64, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x32x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x32x512_32x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 512>,
cutlass::gemm::GemmShape<32, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x32x256_64x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 256>,
cutlass::gemm::GemmShape<64, 32, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x32x512_64x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 512>,
cutlass::gemm::GemmShape<64, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x16x256_32x16x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 256>,
cutlass::gemm::GemmShape<32, 16, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x16x512_32x16x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 512>,
cutlass::gemm::GemmShape<32, 16, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x16x256_16x64x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 256>,
cutlass::gemm::GemmShape<64, 16, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x16x512_16x64x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 512>,
cutlass::gemm::GemmShape<64, 16, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_s8t_s8n_s32t_tensor_op_s32_sparse_sm80.cu
+277 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -55,7 +55,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x256x128_64x64x128)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -74,7 +74,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x128x128_64x64x128)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -93,7 +93,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x128x128_64x64x128)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -113,7 +113,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x64x128_64x64x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -132,7 +132,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x256x128_64x64x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -151,7 +151,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x128x128_32x64x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -170,7 +170,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x64x128_64x32x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -189,7 +189,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x128_32x32x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -208,7 +208,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x128x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -228,7 +228,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x64x256_64x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -247,7 +247,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x256_32x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -261,9 +261,273 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x256_32x32x256) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x128x128_32x32x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x128x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x256x128_32x64x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 128>,
cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x256x256_32x64x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 256>,
cutlass::gemm::GemmShape<32, 64, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 16x128x256_16x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 128, 256>,
cutlass::gemm::GemmShape<16, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 16x128x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 128, 256>,
cutlass::gemm::GemmShape<16, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x32x128_32x32x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x32x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x32x128_64x32x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 128>,
cutlass::gemm::GemmShape<64, 32, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x32x256_64x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 256>,
cutlass::gemm::GemmShape<64, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x16x128_32x16x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 128>,
cutlass::gemm::GemmShape<32, 16, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x16x256_32x16x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 256>,
cutlass::gemm::GemmShape<32, 16, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x16x128_64x16x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 128>,
cutlass::gemm::GemmShape<64, 16, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x16x256_64x16x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 256>,
cutlass::gemm::GemmShape<64, 16, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_testbed_3x.hpp
+17 -10
View File
@@ -346,7 +346,7 @@ struct HostCollectiveMainloop {
tensor_A.resize(a_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(a_coord, stride_factor_A));
tensor_B.resize(b_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(b_coord, stride_factor_B));
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2022));
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2021));
@@ -710,7 +710,7 @@ struct HostCollectiveEpilogue {
using ActivationFunctor = non_void_t<typename FusionOp::ActivationFn,
cutlass::epilogue::thread::Identity<ElementCompute>>;
static constexpr bool IsBiasEnabled = FusionOp::IsPerRowBiasSupported;
static constexpr bool IsRowBiasEnabled = FusionOp::IsPerRowBiasSupported;
static constexpr bool IsDeBiasEnabled = FusionOp::IsDePerRowBiasSupported;
static constexpr bool IsPerRowScaleEnabled = FusionOp::IsPerRowScaleSupported;
static constexpr bool IsScaleFactorEnabled = FusionOp::IsScaleFactorSupported;
@@ -813,6 +813,7 @@ struct HostCollectiveEpilogue {
auto scalar_coord = cutlass::make_Coord(1);
auto col_vector_coord = cutlass::make_Coord(M);
auto row_vector_coord = cutlass::make_Coord(N);
if constexpr (IsPerRowScaleEnabled) {
alpha.resize(col_vector_coord);
EXPECT_TRUE(initialize_tensor(alpha.host_view(), init_scale, seed + 2023));
@@ -849,8 +850,10 @@ struct HostCollectiveEpilogue {
scale_D.sync_device();
}
if constexpr (IsBiasEnabled) {
bias.resize(col_vector_coord);
if constexpr (
IsRowBiasEnabled
) {
bias.resize(IsRowBiasEnabled ? col_vector_coord : row_vector_coord);
EXPECT_TRUE(initialize_tensor(bias.host_view(), init_bias, seed + 2023));
bias.sync_device();
}
@@ -1029,10 +1032,9 @@ struct HostCollectiveEpilogue {
file << "\n\n";
}
if constexpr (IsBiasEnabled) {
if constexpr (IsRowBiasEnabled) {
file << "\n\nBias = \n" << bias.host_view();
}
if constexpr (IsAuxInEnabled) {
file << "\n\nAux Input = \n" << tensor_Aux.host_view();
}
@@ -1090,7 +1092,9 @@ struct HostCollectiveEpilogue {
fusion_args.scale_d_ptr = scale_D.device_data();
}
if constexpr (IsBiasEnabled) {
if constexpr (
IsRowBiasEnabled
) {
fusion_args.bias_ptr = bias.device_data();
}
@@ -1153,7 +1157,7 @@ struct HostCollectiveEpilogue {
auto D = cute::make_tensor(detail::make_iterator(reference_D.host_data()),
cute::make_layout(cute::make_shape(M, N, L), stride_d));
auto Bias = cute::make_tensor(detail::make_iterator(IsDeBiasEnabled ? reference_dbias.host_data() : bias.host_data()),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cute::make_layout(cute::make_shape(IsRowBiasEnabled ? M : N)));
auto Aux = cute::make_tensor(detail::make_iterator(IsAuxInEnabled ? tensor_Aux.host_data() : reference_Aux.host_data()),
cute::make_layout(cute::make_shape(M, N, L), stride_Aux));
auto Valpha = cute::make_tensor(detail::make_iterator(alpha.host_data()),
@@ -1171,7 +1175,8 @@ struct HostCollectiveEpilogue {
decltype(Aux),
decltype(Valpha),
decltype(Vbeta),
ActivationFunctor
ActivationFunctor,
cutlass::plus<ElementCompute>
> epilogue_params{};
epilogue_params.C = C;
@@ -1186,7 +1191,9 @@ struct HostCollectiveEpilogue {
epilogue_params.scale_d = scale_D.at(coord_0);
}
if constexpr (IsBiasEnabled or IsDeBiasEnabled) {
if constexpr (IsRowBiasEnabled
or IsDeBiasEnabled)
{
epilogue_params.Bias = Bias;
}
test/unit/gemm/device/gemm_testbed_3x_ptr_array.hpp
+1792
View File
File diff suppressed because it is too large Load Diff
test/unit/gemm/device/sm90_evt_operations.hpp
+1 -8
View File
@@ -409,14 +409,7 @@ using Sm90LinCombPerColumnBias =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
Sm90RowBroadcast<
ceil_div(
EpilogueDescriptor::StagesC,
size(shape_div(take<0, 2>(typename EpilogueDescriptor::TileShape{}), typename EpilogueDescriptor::EpilogueTile{}))
) + 1,
typename EpilogueDescriptor::TileShape,
ElementBias
>
Sm90RowBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>
>
>;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_aux_store.cu
+4 -8
View File
@@ -130,14 +130,10 @@ bool testEVTAuxStoreWithoutD() {
aux_store_D_block.reset(m * n);
Gemm gemm_op_base;
auto stride_A = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideA{}, cute::make_shape(m, k, cute::Int<1>{}));
auto stride_B = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideB{}, cute::make_shape(n, k, cute::Int<1>{}));
auto stride_C = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideC{}, cute::make_shape(m, n, cute::Int<1>{}));
auto stride_D = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideD{}, cute::make_shape(m, n, cute::Int<1>{}));
auto stride_A = cutlass::make_cute_packed_stride(typename GemmKernel::StrideA{}, {m, k, 1});
auto stride_B = cutlass::make_cute_packed_stride(typename GemmKernel::StrideB{}, {n, k, 1});
auto stride_C = cutlass::make_cute_packed_stride(typename GemmKernel::StrideC{}, {m, n, 1});
auto stride_D = cutlass::make_cute_packed_stride(typename GemmKernel::StrideD{}, {m, n, 1});
auto arguments_base = typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
+120
View File
@@ -0,0 +1,120 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Ptr-Array GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/kernel/tile_scheduler.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/sm70_epilogue_vectorized.hpp"
#include "cutlass/epilogue/collective/default_epilogue.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x_ptr_array.hpp"
#if defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
using namespace cute;
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_256,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayNoSmemWarpSpecialized; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array.cu
+179
View File
@@ -0,0 +1,179 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Ptr-Array GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/kernel/tile_scheduler.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/sm70_epilogue_vectorized.hpp"
#include "cutlass/epilogue/collective/default_epilogue.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x_ptr_array.hpp"
#if defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
using namespace cute;
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_ptr_array, 128x128x64_2x2x1) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC, AlignmentC,
ElementC, LayoutC, AlignmentC,
EpilogueSchedule
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA, AlignmentA,
ElementB, LayoutB, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_ptr_array, 128x128x64_2x2x1_NoSmemEpi) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayNoSmemWarpSpecialized; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC, AlignmentC,
ElementC, LayoutC, AlignmentC,
EpilogueSchedule
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA, AlignmentA,
ElementB, LayoutB, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(TestAll<Gemm>(1.0, 0.0));
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_stream_k_scheduler.cu
+3 -2
View File
@@ -114,13 +114,14 @@ test_scheduler(
<< size<3>(problem_shape_mnkl)
<< " and grid size " << grid.x << "x"
<< grid.y << "x" << grid.z
<< " splits=" << params.splits_
<< " splits=" << params.divmod_splits_.divisor
<< " k_iter=" << params.divmod_tiles_per_output_tile_.divisor
<< " big_units_=" << params.big_units_
<< " big_groups_=" << params.big_groups_
<< " sk_tiles=" << params.sk_tiles_
<< " sk_units=" << params.sk_units_
<< " k_tiles_per_sk_unit=" << params.k_tiles_per_sk_unit_
<< " k_tiles_per_sk_unit=" << params.divmod_k_tiles_per_sk_unit_.divisor
<< " k_tiles_per_sk_big_unit=" << params.divmod_k_tiles_per_sk_big_unit_.divisor
<< " units_per_problem=" << params.units_per_problem_
<< " groups=" << params.divmod_sk_groups_.divisor << std::endl;
};
test/unit/gemm/device/testbed_sparse.h
+22 -21
View File
@@ -128,8 +128,8 @@ struct SparseTestbed {
scope_max = 2;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
scope_min = -2;
scope_max = 1;
scope_min = -1;
} else if (bits_output == 16) {
scope_max = 5;
scope_min = -5;
@@ -353,14 +353,25 @@ struct SparseTestbed {
//
typename Gemm::Arguments arguments{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
tensor_A.device_ref(),
tensor_B.device_ref(),
tensor_C.device_ref(),
tensor_D.device_ref(),
tensor_E_reordered.device_ref(),
split_k_slices,
{alpha, beta},
split_k_slices
tensor_A.device_data(),
tensor_B.device_data(),
tensor_C.device_data(),
tensor_D.device_data(),
tensor_E_reordered.device_data(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
tensor_A.layout().stride(0),
tensor_B.layout().stride(0),
tensor_C.layout().stride(0),
tensor_D.layout().stride(0),
tensor_E_reordered.layout().stride(0)
};
Gemm gemm_op;
@@ -391,7 +402,7 @@ struct SparseTestbed {
bool passed = this->verify(problem_size, alpha, beta);
if (!passed) {
std::cout << "Error with split_k_slices = " << split_k_slices << ", alpha: " << alpha << std::endl;
std::cout << "Error with split_k_slices = " << split_k_slices << ", alpha: " << alpha << ", beta: " << beta << ", m: " << problem_size.m() << ", n: " << problem_size.n() << ", k:" <<problem_size.k() << std::endl;
}
return passed;
@@ -420,11 +431,10 @@ bool TestAllSparseGemm() {
int problem_size_n[] = {kAlignmentN, 512 - 2 * kAlignmentN};
int problem_size_k[] = {Gemm::ThreadblockShape::kK,
Gemm::ThreadblockShape::kK * (Gemm::kStages + 1)};
int problem_size_k[] = {Gemm::ThreadblockShape::kK * 8};
int split_k_slices[] = {
1, 2, 3
1, 2
};
double problem_alpha[] = {
@@ -444,17 +454,8 @@ bool TestAllSparseGemm() {
for (int k : problem_size_k) {
for (int split_k : split_k_slices) {
if (!Gemm::kSplitKSerial && split_k > 1) {
continue;
}
if (split_k > 1 && k / Gemm::ThreadblockShape::kK < split_k) {
continue;
}
for (auto alpha : problem_alpha) {
for (auto beta : problem_beta) {
cutlass::gemm::GemmCoord problem_size(m, n, k);
passed = testbed.run(
test/unit/gemm/device/testbed_with_absmax.h
+66 -27
View File
@@ -212,6 +212,9 @@ struct TestbedWithAmax {
EXPECT_GT(cutlass::reference::host::TensorNorm(underlying_testbed.tensor_D.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), underlying_testbed.tensor_D.host_view());
if (!passed) {
std::cout << "Comparison of D failed" << std::endl;
}
if (kScaleAux) {
tensor_Aux.sync_host();
@@ -219,14 +222,23 @@ struct TestbedWithAmax {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Aux.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(abs_max_Aux.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_Aux.host_view()), 0);
passed &= cutlass::reference::host::TensorEquals(reference_Aux.host_view(), tensor_Aux.host_view());
passed &= cutlass::reference::host::TensorEquals(abs_max_Aux.host_view(), reference_abs_max_Aux.host_view());
if (!cutlass::reference::host::TensorEquals(reference_Aux.host_view(), tensor_Aux.host_view())) {
passed = false;
std::cout << "Comparison of Aux failed" << std::endl;
}
if (!cutlass::reference::host::TensorEquals(abs_max_Aux.host_view(), reference_abs_max_Aux.host_view())) {
passed = false;
std::cout << "Comparison of Aux absmax failed" << std::endl;
}
}
if (kScaleOutput) {
abs_max_D.sync_host();
EXPECT_GT(cutlass::reference::host::TensorNorm(abs_max_D.host_view()), 0);
passed &= cutlass::reference::host::TensorEquals(abs_max_D.host_view(), reference_abs_max_D.host_view());
if (!cutlass::reference::host::TensorEquals(abs_max_D.host_view(), reference_abs_max_D.host_view())) {
passed = false;
std::cout << "Comparison of D absmax failed" << std::endl;
}
}
EXPECT_TRUE(passed) << " mismatched reference";
@@ -417,16 +429,31 @@ struct TestbedWithAmax {
auto arguments = [&]() {
if constexpr (IsSparseTestbed) {
return typename Gemm::Arguments{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
underlying_testbed.tensor_A.device_ref(),
underlying_testbed.tensor_B.device_ref(),
underlying_testbed.tensor_C.device_ref(),
underlying_testbed.tensor_D.device_ref(),
underlying_testbed.tensor_E_reordered.device_ref(),
tensor_Aux.device_ref(),
batch_count,
epilogue_params,
underlying_testbed.tensor_A.device_data(),
underlying_testbed.tensor_B.device_data(),
underlying_testbed.tensor_C.device_data(),
underlying_testbed.tensor_D.device_data(),
underlying_testbed.tensor_E_reordered.device_data(),
tensor_Aux.device_data(),
tensor_Vector.device_data(),
0, // stride vector
epilogue_params
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
underlying_testbed.tensor_A.layout().stride(0),
underlying_testbed.tensor_B.layout().stride(0),
underlying_testbed.tensor_C.layout().stride(0),
underlying_testbed.tensor_D.layout().stride(0),
underlying_testbed.tensor_E_reordered.layout().stride(0),
tensor_Aux.layout().stride(0),
0 // stride vector
};
}
else {
@@ -522,35 +549,47 @@ bool TestAllGemmWithAbsmax(bool scaleA=true, bool scaleB=true, bool scaleC=true)
int M_problems[] = {kAlignmentM, 128 + 32};
int N_problems[] = {kAlignmentN, 512 - 2 * kAlignmentN};
int K_problems[] = {Gemm::ThreadblockShape::kK, Gemm::ThreadblockShape::kK * (Gemm::kStages + 1)};
int K_problems[] = {Gemm::ThreadblockShape::kK * 2};
double alpha_problems[] = {1.};
double beta_problems[] = {0.};
int split_k_slices[] = {
1, 2
};
bool passed = true;
for (int M : M_problems) {
for (int N : N_problems) {
for (int K : K_problems) {
for (double alpha : alpha_problems) {
for (double beta : beta_problems) {
TestbedWithAmax<Gemm, GemmTestbed, ActivationFunctor> testbed(scaleA, scaleB, scaleC);
for (int split_k : split_k_slices) {
if (cutlass::sizeof_bits_v<typename Gemm::EpilogueOutputOp::ElementOutput> <= 8 && split_k > 1) {
// Don't test split-K with FP8 output. The kernel being tested will writie partial accumulations
// for different splits to global memory in FP8, while the reference kernel will not. This leads
// to mismatches that are difficult to capture without a permissive relative equality check threshold.
continue;
}
using ElementAccumulator = typename Gemm::ElementAccumulator;
for (double alpha : alpha_problems) {
for (double beta : beta_problems) {
TestbedWithAmax<Gemm, GemmTestbed, ActivationFunctor> testbed(scaleA, scaleB, scaleC);
passed = testbed.run(
cutlass::gemm::GemmUniversalMode::kGemm,
{M, N, K},
1,
cutlass::from_real<ElementAccumulator>(alpha),
cutlass::from_real<ElementAccumulator>(beta)
);
using ElementAccumulator = typename Gemm::ElementAccumulator;
EXPECT_TRUE(passed)
<< "M: " << M << ", N: " << N << ", K: " << K << ", alpha: " << alpha << ", beta: " << beta;
passed = testbed.run(
cutlass::gemm::GemmUniversalMode::kGemm,
{M, N, K},
split_k,
cutlass::from_real<ElementAccumulator>(alpha),
cutlass::from_real<ElementAccumulator>(beta)
);
if (!passed) {
EXPECT_TRUE(passed)
<< "M: " << M << ", N: " << N << ", K: " << K << ", alpha: " << alpha << ", beta: " << beta << ", split_k:" << split_k;
return passed;
if (!passed) {
return passed;
}
}
}
}
test/unit/gemm/threadblock/mma_multistage_sparse.cu
+105
View File
@@ -179,6 +179,111 @@ TEST(SM80_sparse_gemm_threadblock_congruous,
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_128x32x64_32x32x64_16x8x32_4stage) {
using ElementA = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::half_t;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = float;
using LayoutC = cutlass::layout::ColumnMajor;
cutlass::gemm::GemmCoord problem_size(128, 32, 512);
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 64>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
float alpha = 1.f;
float beta = 0.0f;
int const Stages = 4;
// Define the MmaCore components
using MmaCore = typename cutlass::gemm::threadblock::DefaultSparseMmaCore<
ThreadblockShape, WarpShape, InstructionShape, ElementA, LayoutA,
ElementB, LayoutB, ElementC, LayoutC, cutlass::arch::OpClassTensorOp,
Stages>;
dim3 grid(1, 1);
dim3 block(32, 4, 1);
test::gemm::threadblock::SparseTestbed<MmaCore>(
problem_size.m(), problem_size.n(), problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_32x256x128_32x64x128_16x8x32_4stage) {
using ElementA = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::half_t;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = float;
using LayoutC = cutlass::layout::ColumnMajor;
cutlass::gemm::GemmCoord problem_size(32, 256, 512);
using ThreadblockShape = cutlass::gemm::GemmShape<32, 256, 128>;
using WarpShape = cutlass::gemm::GemmShape<32, 64, 128>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
float alpha = 1.f;
float beta = 0.0f;
int const Stages = 4;
// Define the MmaCore components
using MmaCore = typename cutlass::gemm::threadblock::DefaultSparseMmaCore<
ThreadblockShape, WarpShape, InstructionShape, ElementA, LayoutA,
ElementB, LayoutB, ElementC, LayoutC, cutlass::arch::OpClassTensorOp,
Stages>;
dim3 grid(1, 1);
dim3 block(32, 4, 1);
test::gemm::threadblock::SparseTestbed<MmaCore>(
problem_size.m(), problem_size.n(), problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_128x16x64_32x16x64_16x8x32_4stage) {
using ElementA = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::half_t;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = float;
using LayoutC = cutlass::layout::ColumnMajor;
cutlass::gemm::GemmCoord problem_size(128, 16, 512);
using ThreadblockShape = cutlass::gemm::GemmShape<128, 16, 64>;
using WarpShape = cutlass::gemm::GemmShape<32, 16, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
float alpha = 1.f;
float beta = 0.0f;
int const Stages = 4;
// Define the MmaCore components
using MmaCore = typename cutlass::gemm::threadblock::DefaultSparseMmaCore<
ThreadblockShape, WarpShape, InstructionShape, ElementA, LayoutA,
ElementB, LayoutB, ElementC, LayoutC, cutlass::arch::OpClassTensorOp,
Stages>;
dim3 grid(1, 1);
dim3 block(32, 4, 1);
test::gemm::threadblock::SparseTestbed<MmaCore>(
problem_size.m(), problem_size.n(), problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_128x128x64_64x64x64_16x8x32_4stage) {
using ElementA = cutlass::half_t;
test/unit/gemm/warp/gemm_mixed_input_sm80.cu
+1 -1
View File
@@ -104,7 +104,7 @@ TEST(SM80_warp_gemm_mixed_input_tensor_op_crosswise_i8_f16, 128x128x64_64x64x64_
using Shape = cutlass::gemm::GemmShape<64, 64, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 16>;
using ElementA = int8_t;
using ElementB = cutlass::half_t;;
using ElementB = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<ElementA>::value, 64>;
test/unit/gemm/warp/gemm_sparse_sm80.cu
+60 -18
View File
@@ -327,6 +327,48 @@ TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 128x128x64_32x32x64_16x8x32)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 64x32x64_32x32x64_16x8x32) {
using Shape = cutlass::gemm::GemmShape<32, 32, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
using Element = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 32>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 32, 64> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 64x16x64_32x16x64_16x8x32) {
using Shape = cutlass::gemm::GemmShape<32, 16, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
using Element = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 16>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 16, 64> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 128x64x128_64x32x128_16x8x32) {
using Shape = cutlass::gemm::GemmShape<64, 32, 128>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
@@ -402,7 +444,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_64x64x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -423,7 +465,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_64x32x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -444,7 +486,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_32x64x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -465,7 +507,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_32x32x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -486,7 +528,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_32x16x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -507,7 +549,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x64x256_64x32x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 64, 256> >()
@@ -528,7 +570,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 64x128x256_32x64x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 128, 256> >()
@@ -549,7 +591,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 64x64x256_32x32x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 64, 256> >()
@@ -570,7 +612,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 64x32x256_32x16x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 32, 256> >()
@@ -927,7 +969,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_64x64x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -948,7 +990,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_64x32x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -969,7 +1011,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_32x64x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -990,7 +1032,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_32x32x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -1011,7 +1053,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_32x16x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -1032,7 +1074,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x64x512_64x32x512_16x8x128
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 64, 512> >()
@@ -1053,7 +1095,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 64x128x512_32x64x512_16x8x128
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 128, 512> >()
@@ -1074,7 +1116,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 64x64x512_32x32x512_16x8x128)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 64, 512> >()
@@ -1095,7 +1137,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 64x32x512_32x16x512_16x8x128)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 32, 512> >()
test/unit/nvrtc/kernel/thread/contraction.hpp
+6 -6
View File
@@ -49,16 +49,16 @@ struct ContractionKernel {
using ElementScalar = float;
using ElementAccum = float;
using EpilogueThread = cutlass::epilogue::thread::LinearCombination<ElementC,
1,
ElementAccum,
ElementScalar>;
using EpilogueThread = cutlass::epilogue::thread::LinearCombination<ElementC,
1,
ElementAccum,
ElementScalar>;
static constexpr cute::GMMA::Major majorA = ! kTransA ? cute::GMMA::Major::MN : cute::GMMA::Major::K;
static constexpr cute::GMMA::Major majorB = ! kTransB ? cute::GMMA::Major::K : cute::GMMA::Major::MN;
/// Kernel config
typedef int64_t stride_type;
typedef int64_t stride_type;
typedef int32_t extent_type;
static constexpr const stride_type* stride_null = nullptr;
@@ -117,7 +117,7 @@ using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
using EpilogueOutputOp = cutlass::epilogue::collective::DefaultEpilogue<StrideC, StrideC, EpilogueThread, cutlass::gemm::EpilogueDefault>;
using CollectiveEpilogue = cutlass::epilogue::collective::detail::Sm90TmaWarpSpecializedAdapter<EpilogueOutputOp>;
using Kernel = cutlass::gemm::kernel::GemmUniversal<
ProblemShape,
ProblemShape,
CollectiveOp,
CollectiveEpilogue>;
test/unit/transform/CMakeLists.txt
+5 -2
View File
@@ -27,15 +27,18 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add_subdirectory(threadblock)
add_subdirectory(kernel)
add_custom_target(
cutlass_test_unit_transform
DEPENDS
cutlass_test_unit_transform_threadblock
)
cutlass_test_unit_transform_filter_format
)
add_custom_target(
test_unit_transform
DEPENDS
test_unit_transform_threadblock
)
test_unit_transform_kernel
)
test/unit/transform/kernel/CMakeLists.txt
+32
View File
@@ -0,0 +1,32 @@
# Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cutlass_test_unit_add_executable(
cutlass_test_unit_transform_filter_format
filter_format_transformer.cu
)
test/unit/transform/kernel/filter_format_transformer.cu
+261
View File
@@ -0,0 +1,261 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests cutlass::transform::kernel::ConvFilterFormatTransformer
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/transform/pitch_linear_thread_map.h"
#include "cutlass/transform/kernel/filter_format_transformer.hpp"
#include "cutlass/transform/device/transform_universal_adapter.hpp"
#include "thrust/universal_vector.h"
#include "thrust/host_vector.h"
#include "thrust/device_vector.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
template <class Element, class Shape_S>
auto verify_ckrs_to_crsk(thrust::host_vector<Element> const &S, thrust::host_vector<Element> const &D, Shape_S shape_s) {
using namespace cute;
int32_t errors = 0;
int32_t const kErrorLimit = 10;
if (S.size() != D.size()) {
return false;
}
auto shape_d = select<2, 0, 1, 3>(shape_s);
for (int i = 0; i < (int)S.size(); ++i) {
auto [s, r, k, c] = idx2crd(i, shape_s);
auto d_idx = crd2idx(make_coord(k, s, r, c), shape_d);
if (S[i] != D[d_idx]) {
std::cerr << "Error. S[" << i << "]: " << S[i] << ", D[" << d_idx << "]: " << D[d_idx] << std::endl;
if (++errors >= kErrorLimit) {
std::cerr << "Aborting on " << kErrorLimit << "nth error." << std::endl;
return false;
}
}
}
return errors == 0;
}
template <class Element, class Shape_S>
auto verify_ckrs_to_krsc(thrust::host_vector<Element> const &S, thrust::host_vector<Element> const &D, Shape_S shape_s) {
using namespace cute;
int32_t errors = 0;
int32_t const kErrorLimit = 10;
if (S.size() != D.size()) {
return false;
}
auto shape_d = select<3, 0, 1, 2>(shape_s);
for (int i = 0; i < (int)S.size(); ++i) {
auto [s, r, k, c] = idx2crd(i, shape_s);
auto d_idx = crd2idx(make_coord(c, s, r, k), shape_d);
if (S[i] != D[d_idx]) {
std::cerr << "Error. S[" << i << "]: " << S[i] << ", D[" << d_idx << "]: " << D[d_idx] << std::endl;
if (++errors >= kErrorLimit) {
std::cerr << "Aborting on " << kErrorLimit << "nth error." << std::endl;
return false;
}
}
}
return errors == 0;
}
template <class Element,
cutlass::transform::kernel::FilterFormat SrcFormat,
cutlass::transform::kernel::FilterFormat DstFormat,
int Alignment = 16>
bool transform_test() {
using namespace cute;
using TransformKernel = cutlass::transform::kernel::ConvFilterFormatTransformer<SrcFormat, DstFormat, 4, Element, Alignment>;
using Transform = cutlass::transform::device::TransformUniversalAdapter<TransformKernel>;
auto s = 3;
auto r = 3;
auto k = 64 + Alignment / (int)(sizeof(Element));
auto c = 64 + Alignment / (int)(sizeof(Element));
thrust::host_vector<Element> h_S(s * r * k * c);
thrust::host_vector<Element> h_D(s * r * k * c);
//
// Initialize
//
for (int i = 0; i < (int)h_S.size(); ++i) {
h_S[i] = static_cast<Element>(i);
h_D[i] = Element{};
}
thrust::device_vector<Element> d_S = h_S;
thrust::device_vector<Element> d_D = h_D;
Transform transform_op;
const void* src_ptr = static_cast<const void *>(d_S.data().get());
void* dst_ptr = static_cast<void *>(d_D.data().get());
typename TransformKernel::FilterExtent filter_extent;
filter_extent[0] = k;
filter_extent[1] = r;
filter_extent[2] = s;
filter_extent[3] = c;
auto args = typename Transform::Arguments {
src_ptr,
dst_ptr,
filter_extent
};
cutlass::Status status = cutlass::Status::kInvalid;
size_t workspace_size = Transform::get_workspace_size(args);
thrust::universal_vector<uint8_t> workspace(workspace_size);
status = transform_op.initialize(args, workspace.data().get());
if (status != cutlass::Status::kSuccess) {
cudaError_t error = cudaGetLastError();
std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
return false;
}
status = transform_op();
EXPECT_TRUE(status == cutlass::Status::kSuccess);
if (status != cutlass::Status::kSuccess) {
return false;
}
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess) << " Kernel execution error: "
<< cudaGetErrorString(result);
// Verification
h_D = d_D;
auto tensor_shape_S = make_shape(s, r, k, c);
bool passed = false;
if constexpr(DstFormat == cutlass::transform::kernel::FilterFormat::KTRSC) {
// KTRSC
passed = verify_ckrs_to_krsc(h_S, h_D, tensor_shape_S);
}
else if constexpr(DstFormat == cutlass::transform::kernel::FilterFormat::CTRSK) {
// CTRSK;
passed = verify_ckrs_to_crsk(h_S, h_D, tensor_shape_S);
}
return passed;
}
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
TEST(Transform_kernel_ConvFilterFormatTransformer, ckrs_to_crsk) {
bool passed = true;
// fp16 kernel with alignment bytes from 16 to 2.
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK>();
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 8>();
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 4>();
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 2>();
// fp8 kernel with alignment bytes from 16 to 1.
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 8>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 4>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 2>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 1>();
// int8 kernel with alignment bytes from 16 to 1.
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 8>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 4>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 2>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 1>();
// fp32 kernel with alignment bytes from 16 to 4.
passed &= transform_test<float, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK>();
passed &= transform_test<float, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 8>();
passed &= transform_test<float, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::CTRSK, 4>();
EXPECT_TRUE(passed);
}
// CKRS -> KRSC
TEST(Transform_kernel_ConvFilterFormatTransformer, ckrs_to_krsc) {
bool passed = true;
// fp16 kernel with alignment bytes from 16 to 2.
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC>();
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 8>();
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 4>();
passed &= transform_test<cutlass::half_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 2>();
// fp8 kernel with alignment bytes from 16 to 1.
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 8>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 4>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 2>();
passed &= transform_test<cutlass::float_e4m3_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 1>();
// int8 kernel with alignment bytes from 16 to 1.
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 8>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 4>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 2>();
passed &= transform_test<int8_t, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 1>();
// fp32 kernel with alignment bytes from 16 to 4.
passed &= transform_test<float, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC>();
passed &= transform_test<float, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 8>();
passed &= transform_test<float, cutlass::transform::kernel::FilterFormat::CKTRS, cutlass::transform::kernel::FilterFormat::KTRSC, 4>();
EXPECT_TRUE(passed);
}
#endif