leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

CUTLASS 3.5.1 (#1623)

Browse Source 
* CUTLASS 3.5.1

* updates, optimizations, fixes
This commit is contained in:
Vijay Thakkar
2024-07-29 08:46:24 -04:00
committed by GitHub
parent 56b46e2d13
commit be60a0b272
312 changed files with 19793 additions and 6775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/conv/device/conv2d_fprop_implicit_gemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32_sm89.cu
+7 -7
View File
@@ -60,7 +60,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc_
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -104,7 +104,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe5m2nhwc_fe4m3nhwc_fe4m3nhw
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -148,7 +148,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe5m2nhwc_fe4m3nhwc_fe5m2nhw
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e5m2_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -192,7 +192,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -236,7 +236,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -280,7 +280,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -324,7 +324,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
test/unit/conv/device/conv2d_fprop_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Conv2dFpropSM80TestResidaulBlock() {
static void Conv2dFpropSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Conv2dFpropSM80TestResidaulBlock() {
TEST(SM80_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Conv2dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_fprop_with_broadcast_sm70.cu
+2 -2
View File
@@ -60,7 +60,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = false
>
void TestResidaulBlock() {
void Conv2dFpropSM70TestResidualBlock() {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
@@ -117,7 +117,7 @@ void TestResidaulBlock() {
TEST(SM70_Device_Conv2d_Fprop_With_Residual_Block_Plus_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// Resnet
TestResidaulBlock<float, cutlass::epilogue::thread::ReLu, cutlass::plus, cutlass::epilogue::thread::Identity>();
Conv2dFpropSM70TestResidualBlock<float, cutlass::epilogue::thread::ReLu, cutlass::plus, cutlass::epilogue::thread::Identity>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_fprop_with_broadcast_sm75.cu
+3 -3
View File
@@ -103,7 +103,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
void Conv2dFpropSM75TestResidualBlock() {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
@@ -160,14 +160,14 @@ void TestResidaulBlock() {
TEST(SM75_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// Resnet
TestResidaulBlock<cutlass::half_t, cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM75TestResidualBlock<cutlass::half_t, cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
TEST(SM75_Device_Conv2d_Fprop_With_Residual_Block_Multiply_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// EfficientNet V2
// Do not run split-K tests since the activation op is not Identity.
TestResidaulBlock<float, cutlass::epilogue::thread::Sigmoid, cutlass::multiplies, cutlass::epilogue::thread::Identity, false>();
Conv2dFpropSM75TestResidualBlock<float, cutlass::epilogue::thread::Sigmoid, cutlass::multiplies, cutlass::epilogue::thread::Identity, false>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_fprop_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Conv3dFpropSM80TestResidaulBlock() {
static void Conv3dFpropSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Conv3dFpropSM80TestResidaulBlock() {
TEST(SM80_Device_Conv3d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Conv3dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv3dFpropSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_with_broadcast_testbed.h
+37 -20
View File
@@ -204,17 +204,29 @@ public:
}
void initialize(
cutlass::conv::Conv3dProblemSize const &problem_size, uint64_t seed = 2019) {
cutlass::conv::Conv3dProblemSize const &problem_size, bool non_packed_test = false, uint64_t seed = 2019) {
tensor_A.resize(implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size));
tensor_B.resize(implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size));
tensor_C.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_C_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Z_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Z_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
// to make the layout of tensors a little bit bigger than the problem size
cutlass::Tensor5DCoord stride_increment = cutlass::Tensor5DCoord(8, 16, 32, 32, 64);
cutlass::Tensor5DCoord tensor_A_extent = implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size);
cutlass::Tensor5DCoord tensor_B_extent = implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size);
cutlass::Tensor5DCoord tensor_C_extent = implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size);
if (non_packed_test) {
tensor_A_extent += stride_increment;
tensor_C_extent += stride_increment;
}
tensor_A.resize(tensor_A_extent);
tensor_B.resize(tensor_B_extent);
tensor_C.resize(tensor_C_extent);
tensor_C_reference.resize(tensor_C_extent);
tensor_Z_computed.resize(tensor_C_extent);
tensor_Z_reference.resize(tensor_C_extent);
tensor_T_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_T_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Y_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Y_reference.resize(tensor_C_extent);
tensor_Broadcast.resize({
1,
1,
@@ -282,6 +294,7 @@ public:
bool run(
cutlass::conv::Conv3dProblemSize const &problem_size,
cutlass::conv::SplitKMode const &split_k_mode = cutlass::conv::SplitKMode::kSerial,
bool non_packed_test = false,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(1)) {
@@ -300,7 +313,7 @@ public:
<< std::endl;
#endif
initialize(problem_size);
initialize(problem_size, non_packed_test);
// configure the operator
Conv3d conv3d_op;
@@ -479,6 +492,7 @@ public:
<< problem_size.dilation_h << "x"
<< problem_size.dilation_w << "_"
<< (problem_size.mode == cutlass::conv::Mode::kCrossCorrelation ? "xcorr_" : "conv_")
<< (non_packed_test ? "non_packed_tensor_test_" : "packed_tensor_test_")
<< Conv3d::ThreadblockShape::kM << "x"
<< Conv3d::ThreadblockShape::kN << "x"
<< Conv3d::ThreadblockShape::kK << "_"
@@ -521,7 +535,8 @@ template <typename ImplicitGemm,
>
bool TestAllConv3dWithBroadcast(
const Conv3dProblemVector &conv_test_sizes = Conv3dProblemVector(),
const Conv3dProblemVector &conv_blacklist_sizes = Conv3dProblemVector()) {
const Conv3dProblemVector &conv_blacklist_sizes = Conv3dProblemVector(),
bool non_packed_test = false) {
bool passed = true;
@@ -595,17 +610,17 @@ bool TestAllConv3dWithBroadcast(
// test mode = xcross
passed = testbed.run(
conv_problem,
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
// test mode = convolution
passed = testbed.run(
conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
@@ -651,6 +666,7 @@ bool TestAllConv3dWithBroadcast(
passed = testbed.run(
conv3d_split_k_test_size.reset_split_k_slices(split_k_slice),
split_k_mode,
false,/*non_packed_test*/
cutlass::from_real<typename ImplicitGemm::ElementCompute>(alpha),
cutlass::from_real<typename ImplicitGemm::ElementCompute>(beta));
@@ -669,7 +685,8 @@ template <typename ImplicitGemm,
typename ReferenceOp = Conv3dWithBroadcastReferenceOp<ImplicitGemm>,
bool AddBroadcastFirst = false>
bool TestSpecificConv3dWithBroadcast(
const Conv3dProblemVector & problem_sizes) {
const Conv3dProblemVector & problem_sizes,
bool non_packed_test = false) {
bool passed = true;
@@ -686,19 +703,19 @@ bool TestSpecificConv3dWithBroadcast(
// Test
//
// test mode = xcross
// test mode = xcross, non_packed_test = false
passed = testbed.run(
conv_problem,
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
// test mode = convolution
// test mode = convolution, non_packed_test = false
passed = testbed.run(
conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
test/unit/conv/device/deconv2d_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv2dSM80TestResidaulBlock() {
static void Deconv2dSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -163,7 +163,7 @@ static void Deconv2dSM80TestResidaulBlock() {
TEST(SM80_Device_Deconv2d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv2dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Deconv2dSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/deconv3d_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -103,7 +103,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv3dSM80TestResidaulBlock() {
static void Deconv3dSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Deconv3dSM80TestResidaulBlock() {
TEST(SM80_Device_Deconv3d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv3dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Deconv3dSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device_3x/conv_problem_sizes.hpp
+181 -9
View File
@@ -42,7 +42,7 @@ namespace test::conv::device {
/////////////////////////////////////////////////////////////////////////////////////////////////
template<int SpatialDim, cutlass::conv::Operator ConvOp>
template<int SpatialDim, cutlass::conv::Operator ConvOp, bool SupportStrides = (ConvOp != cutlass::conv::Operator::kDgrad)>
std::vector<cutlass::conv::ConvProblemShape<ConvOp, SpatialDim>>
inline
get_conv_problem_vector();
@@ -297,7 +297,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kFprop>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 8, 8, 64},
{2, 7, 7, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
@@ -319,7 +319,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kFprop>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 64},
{2, 16, 15, 64},
{256, 2, 5, 64},
{1, 1},
{0, 0},
@@ -658,7 +658,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 32},
{2, 15, 16, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
@@ -680,7 +680,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
// 2x5 filter, asymmetric padding 1,0/1,0, w/ stride, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 16, 32},
{2, 16, 15, 32},
{256, 2, 5, 32},
{1, 1},
{0, 0},
@@ -688,6 +688,28 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kWgrad>() {
{2, 3},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 64, 16, 128}, // nhwc
{640, 1, 1, 128}, // krsc
{0, 0}, // padding lower (pad_h, pad_w)
{0, 0}, // padding upper (pad_h, pad_w)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
1 // group
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 65, 16, 128}, // nhwc
{640, 1, 1, 128}, // krsc
{0, 0}, // padding lower (pad_h, pad_w)
{0, 0}, // padding upper (pad_h, pad_w)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
1 // group
});
return problem_shapes;
}
@@ -751,17 +773,39 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>() {
{2, 2, 3},
1
});
// To test streamk, equals to gemm-MxNxK size 128x640x2048
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 64, 16, 128}, // ndhwc
{640, 1, 1, 1, 128}, // ktrsc
{0, 0, 0}, // padding lower (pad_d, pad_h, pad_w)
{0, 0, 0}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
1 // group
});
// To test streamk, equals to gemm-MxNxK size 128x640x2080
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 1, 65, 16, 128}, // ndhwc
{640, 1, 1, 1, 128}, // ktrsc
{0, 0, 0}, // padding lower (pad_d, pad_h, pad_w)
{0, 0, 0}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
1 // group
});
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Dgrad
// Unit Stride Dgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
// Specialization for 1D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>> inline
get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad>() {
get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, false>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>;
std::vector<ProblemShape> problem_shapes;
problem_shapes.push_back({
@@ -884,7 +928,7 @@ get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad>() {
// Specialization for 2D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>> inline
get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad>() {
get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, false>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>;
std::vector<ProblemShape> problem_shapes;
problem_shapes.push_back({
@@ -1007,7 +1051,7 @@ get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad>() {
// Specialization for 3D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>> inline
get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad>() {
get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad, false>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>;
std::vector<ProblemShape> problem_shapes;
// Filter-K = 16 for predication
@@ -1082,6 +1126,134 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad>() {
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Strided Dgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
// Specialization for 1D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>> inline
get_conv_problem_vector<1, cutlass::conv::Operator::kDgrad, true>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 1>;
std::vector<ProblemShape> problem_shapes;
// non-packed input/output strides.
// stride divides dilation
// asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 8, 64}, // nqk
{800, 80, 1}, // stride (nqk)
{64, 3, 64}, // ksc
{64, 64, 1}, // stride (ksc)
{800, 80, 1}, // stride (nwc)
{0}, // padding lower (pad_w)
{1}, // padding upper (pad_w)
{2}, // stride (stride_w)
{4}, // dilation (dilation_w)
1 // group
});
// non-packed input/output strides.
// dilation divides stride
// asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 8, 64}, // nqk
{800, 80, 1}, // stride (nqk)
{64, 3, 64}, // ksc
{64, 64, 1}, // stride (ksc)
{800, 80, 1}, // stride (nwc)
{1}, // padding lower (pad_w)
{0}, // padding upper (pad_w)
{4}, // stride (stride_w)
{2}, // dilation (dilation_w)
1 // group
});
// non-packed input/output strides.
// stride dilation dont divide
// asymmetric padding
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 8, 64}, // nqk
{800, 80, 1}, // stride (nqk)
{64, 3, 64}, // ksc
{64, 64, 1}, // stride (ksc)
{800, 80, 1}, // stride (nwc)
{1}, // padding lower (pad_w)
{2}, // padding upper (pad_w)
{2}, // stride (stride_w)
{3}, // dilation (dilation_w)
1 // group
});
return problem_shapes;
}
// Specialization for 2D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>> inline
get_conv_problem_vector<2, cutlass::conv::Operator::kDgrad, true>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 2>;
std::vector<ProblemShape> problem_shapes;
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
// mode 0 stride divides dilation
// mode 1 dilation divides stride
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 16, 16, 64},
{256, 2, 5, 64},
{1, 0},
{0, 1},
{2, 4},
{4, 2},
1
});
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
// mode 0 dilation divides stride
// mode 1 stride divides dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 16, 16, 64},
{256, 2, 5, 64},
{1, 0},
{0, 1},
{4, 2},
{2, 4},
1
});
// 2x5 filter, asymmetric padding 1,0/1,0, w/ dilation
// stride dilation dont divide
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{3, 16, 16, 64},
{256, 2, 5, 64},
{1, 0},
{0, 1},
{3, 2},
{2, 3},
1
});
return problem_shapes;
}
// Specialization for 3D dgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>> inline
get_conv_problem_vector<3, cutlass::conv::Operator::kDgrad, true>() {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kDgrad, 3>;
std::vector<ProblemShape> problem_shapes;
// Filter 3x4x5 + asymmetric padding 102/010, w/ dilation
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{2, 16, 10, 16, 64},
{64, 3, 4, 5, 96},
{1, 0, 1},
{0, 2, 0},
{2, 4, 2},
{4, 2, 3},
1
});
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace cutlass::test
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_f16_f16_f32_tensorop_f16.cu
+8
View File
@@ -93,6 +93,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -137,6 +138,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -181,6 +183,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -225,6 +228,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -273,6 +277,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -317,6 +322,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -361,6 +367,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -405,6 +412,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f16
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_f16_f16_f32_tensorop_f32.cu
+8
View File
@@ -93,6 +93,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -137,6 +138,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -181,6 +183,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -225,6 +228,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -273,6 +277,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -317,6 +322,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -361,6 +367,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -405,6 +412,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/conv/device_3x/fprop/sm90_conv2d_fprop_implicit_gemm_tf32_tf32_f32_tensorop_f32.cu
+8
View File
@@ -93,6 +93,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -137,6 +138,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -181,6 +183,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -225,6 +228,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -273,6 +277,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -317,6 +322,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -361,6 +367,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
//
@@ -405,6 +412,7 @@ TEST(SM90_device_conv2d_fprop_implicitgemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(/*alpha=*/1.0, /*beta=*/1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/conv/device_3x/testbed_conv.hpp
+134 -32
View File
@@ -40,6 +40,7 @@
#include "cutlass/kernel_hardware_info.hpp"
#include "cutlass/conv/convolution.h"
#include "cutlass/conv/convnd_problem_shape.hpp"
#include "../test/unit/gemm/device/gemm_testbed_3x.hpp"
#include "thrust/universal_vector.h"
#include "cutlass/util/distribution.h"
@@ -64,6 +65,7 @@
namespace test::conv::device {
/////////////////////////////////////////////////////////////////////////////////////////////////
// Initializes a flat device buffer
template <typename Element>
static void
@@ -104,7 +106,39 @@ initialize_values(
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// utils for sparse or dense conv parameters
template <class Conv>
struct DenseConvParams {
// Default Kernel data types
using ElementA = typename Conv::ConvKernel::ElementA;
using ElementB = typename Conv::ConvKernel::ElementB;
static constexpr cutlass::conv::Operator ConvOp = Conv::DispatchPolicy::ConvOp;
static constexpr int NumSpatialDimensions = Conv::NumSpatialDimensions;
using ProblemShape = cutlass::conv::ConvProblemShape<ConvOp, NumSpatialDimensions>;
// get the default arguments without sparse data
auto get_mainloop_arguments(
ProblemShape const& problem_shape,
thrust::universal_vector<ElementA>& tensor_A,
thrust::universal_vector<ElementB>& tensor_B
) {
auto args = typename Conv::ConvKernel::MainloopArguments {
problem_shape,
tensor_A.data().get(),
tensor_B.data().get(),
};
return args;
}
};
template <class Conv>
struct SparseConvParams {
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <class Conv, bool isSparseEnabled_ = false>
struct ConvTestbed {
// Kernel data types
using ElementA = typename Conv::ConvKernel::ElementA;
@@ -114,6 +148,11 @@ struct ConvTestbed {
using ElementD = typename Conv::ConvKernel::ElementD;
using ElementAccumulator = typename Conv::ConvKernel::ElementAccumulator;
// ConvTest for sparse kernel
static constexpr bool isSparseEnabled = isSparseEnabled_;
using ConvParams = cute::conditional_t<isSparseEnabled, SparseConvParams<Conv>, DenseConvParams<Conv>>;
ConvParams params;
//
// FusionOperation derived types/queries
//
@@ -134,6 +173,8 @@ struct ConvTestbed {
static constexpr bool IsBiasEnabled = cutlass::epilogue::collective::detail::IsThreadEpilogueOpWithBias<FusionOp>::value &&
!cute::is_same_v<BiasType, void>;
static constexpr bool DisableSource = cute::is_void_v<typename FusionOp::ElementSource>;
using StrideC = typename Conv::ConvKernel::StrideC;
using StrideD = typename Conv::ConvKernel::StrideD;
using ThreadEpilogueOp = typename Conv::ConvKernel::CollectiveEpilogue::ThreadEpilogueOp;
@@ -141,6 +182,10 @@ struct ConvTestbed {
static constexpr cutlass::conv::Operator ConvOp = Conv::DispatchPolicy::ConvOp;
static constexpr int NumSpatialDimensions = Conv::NumSpatialDimensions;
using ProblemShape = cutlass::conv::ConvProblemShape<ConvOp, NumSpatialDimensions>;
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
using MaxSwizzleSize = typename gemm::device::detail::MaxSwizzleSize;
using Splits = typename gemm::device::detail::Splits;
using Schedule = typename Conv::DispatchPolicy::Schedule;
/// Initialization
@@ -148,6 +193,7 @@ struct ConvTestbed {
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform;
cutlass::Distribution::Kind init_C = cutlass::Distribution::Uniform;
cutlass::Distribution::Kind init_bias = cutlass::Distribution::Uniform;
cutlass::Distribution::Kind init_disable = cutlass::Distribution::Identity; // all zeros
uint64_t seed = 6090;
float epsilon = 0.0f;
int split_p_slices = 1;
@@ -160,7 +206,8 @@ struct ConvTestbed {
thrust::universal_vector<ElementScalar> tensor_alpha;
thrust::universal_vector<ElementScalar> tensor_beta;
void initialize(ProblemShape const& problem_shape, uint64_t seed = 6090) {
// Return true on success, else false
bool initialize(ProblemShape const& problem_shape, uint64_t seed = 6090) {
tensor_A.resize(sizeof(ElementA) * problem_shape.size_A());
tensor_B.resize(sizeof(ElementB) * problem_shape.size_B());
tensor_C.resize(sizeof(ElementC) * problem_shape.size_C());
@@ -171,6 +218,12 @@ struct ConvTestbed {
initialize_values(tensor_B, init_B, seed * 11);
initialize_values(tensor_C, init_C, seed * 17);
initialize_values(tensor_bias, init_bias, seed * 19);
bool flag = true;
if constexpr (isSparseEnabled) {
flag &= params.initialize(problem_shape, tensor_B, static_cast<int>(seed + 2023));
}
return flag;
}
// Determine SMEM requirements and waive if not satisfied
@@ -190,11 +243,16 @@ struct ConvTestbed {
return max_smem_size >= Conv::ConvKernel::SharedStorageSize;
}
/// Executes one test
// Executes one test
bool run(
ProblemShape const& problem_shape,
ElementScalar alpha = ElementScalar(1),
ElementScalar beta = ElementScalar(0)
,
RasterOrderOptions raster_order = RasterOrderOptions::Heuristic,
MaxSwizzleSize max_swizzle = MaxSwizzleSize{},
Splits splits = Splits{},
DecompositionMode decomposition_mode = DecompositionMode::Heuristic
) {
// Waive test if insufficient CUDA device
@@ -205,7 +263,12 @@ struct ConvTestbed {
return true;
}
initialize(problem_shape);
bool ret = initialize(problem_shape);
if (!ret) {
std::cerr << "initialize failed for the given problem_shape: \n";
return false;
}
cutlass::KernelHardwareInfo hw_info;
cudaGetDevice(&hw_info.device_id);
@@ -230,20 +293,27 @@ struct ConvTestbed {
cute::get<0, i>(stride_D) = problem_shape.stride_C[ProblemShape::RankT-2-i];
});
}
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
typename Conv::ConvKernel::TileScheduler::Arguments scheduler_args{};
if constexpr (cute::is_same_v<typename Conv::ConvKernel::TileScheduler::Arguments, cutlass::gemm::StreamKScheduler>) {
scheduler_args = { static_cast<int>(splits), static_cast<int>(max_swizzle), raster_order, decomposition_mode };
}
auto mainloop_args = params.get_mainloop_arguments(problem_shape, tensor_A, tensor_B);
auto epilogue_args = typename Conv::ConvKernel::EpilogueArguments {
{},
tensor_C.data().get(),
stride_C,
tensor_D_computed.data().get(),
stride_D,
};
auto args = typename Conv::Arguments {
{
problem_shape,
tensor_A.data().get(),
tensor_B.data().get(),
}, // MainloopArguments
{
{},
tensor_C.data().get(),
stride_C,
tensor_D_computed.data().get(),
stride_D,
}, // EpilogueArguments
mainloop_args, // MainloopArguments
epilogue_args, // EpilogueArguments
hw_info,
scheduler_args
};
@@ -462,6 +532,8 @@ struct ConvTestbed {
for (size_t i = 0; i < size_t(size(reference)); ++i) {
if (reference(i) != computed(i)) {
passed = false;
printf("[%llu] %f, %f\n", static_cast<unsigned long long>(i),
float(reference(i)), float(computed(i)));
break;
}
}
@@ -475,6 +547,8 @@ struct ConvTestbed {
if (std::isnan(abs_error) || std::isnan(rel_error) ||
std::min(abs_error, rel_error) > epsilon) {
passed = false;
printf("[%llu] %f, %f\n", static_cast<unsigned long long>(i),
float(reference(i)), float(computed(i)));
break;
}
}
@@ -488,18 +562,20 @@ struct ConvTestbed {
cute::print("\n");
for (size_t i = 0; i < size_t(size(A)); ++i) {
printf("[%ld]: A = %f\n", i, float(A(i)));
printf("[%llu]: A = %f\n", static_cast<unsigned long long>(i), float(A(i)));
}
for (size_t i = 0; i < size_t(size(B)); ++i) {
printf("[%ld]: B = %f\n", i, float(B(i)));
printf("[%llu]: B = %f\n", static_cast<unsigned long long>(i), float(B(i)));
}
if constexpr (IsBiasEnabled) {
for (size_t i = 0; i < size_t(size(tensor_bias)); ++i) {
printf("[%ld]: bias = %f\n", i, float(tensor_bias(i)));
printf("[%llu]: bias = %f\n", static_cast<unsigned long long>(i),
float(tensor_bias(i)));
}
}
for (size_t i = 0; i < size_t(size(reference)); ++i) {
printf("[%ld]: ref = %f, computed = %f\n", i, float(reference(i)), float(computed(i)));
printf("[%llu]: ref = %f, computed = %f\n", static_cast<unsigned long long>(i),
float(reference(i)), float(computed(i)));
}
}
#endif
@@ -509,30 +585,56 @@ struct ConvTestbed {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Conv>
bool TestAllConv(double alpha = 1.0, double beta = 0.0, float epsilon = 0.0f) {
template <typename Conv, bool SupportStrides = (Conv::DispatchPolicy::ConvOp != cutlass::conv::Operator::kDgrad)>
bool TestAllConv(double alpha = 1.0, double beta = 0.0, float epsilon = 0.0f
) {
using ElementScalar = typename Conv::EpilogueOutputOp::ElementScalar;
bool passed = true;
ConvTestbed<Conv> testbed;
testbed.epsilon = epsilon;
auto problem_vector = get_conv_problem_vector<
Conv::NumSpatialDimensions, Conv::DispatchPolicy::ConvOp>();
Conv::NumSpatialDimensions, Conv::DispatchPolicy::ConvOp, SupportStrides>();
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using MaxSwizzleSize = typename gemm::device::detail::MaxSwizzleSize;
using Splits = typename gemm::device::detail::Splits;
std::vector<DecompositionMode> decomposition_modes = {DecompositionMode::Heuristic};
static constexpr bool UsesStreamKScheduler = cute::is_same_v<typename Conv::ConvKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>;
if constexpr (UsesStreamKScheduler) {
decomposition_modes.push_back(DecompositionMode::DataParallel);
decomposition_modes.push_back(DecompositionMode::SplitK);
decomposition_modes.push_back(DecompositionMode::StreamK);
}
for (auto conv_problem : problem_vector) {
#if CUTLASS_DEBUG_TRACE_LEVEL > 0
print(conv_problem);
print(conv_problem);
#endif
for (DecompositionMode decomp_mode : decomposition_modes) {
std::vector problem_splits = {Splits{1}};
if (decomp_mode == DecompositionMode::Heuristic || decomp_mode == DecompositionMode::SplitK) {
problem_splits.push_back(Splits{2});
}
for (auto splits : problem_splits) {
passed = testbed.run(
conv_problem,
cutlass::from_real<ElementScalar>(alpha),
cutlass::from_real<ElementScalar>(beta));
if (!passed) {
printf("Failed test for "); print(conv_problem);
return false;
}
passed = testbed.run(
conv_problem,
cutlass::from_real<ElementScalar>(alpha),
cutlass::from_real<ElementScalar>(beta)
,RasterOrderOptions::Heuristic, // raster_order
MaxSwizzleSize(1),
splits,
decomp_mode
);
if (!passed) {
printf("Failed test for "); print(conv_problem);
return false;
}
} // splits
} // decomposition_mode
}
return passed;