leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

CUTLASS 3.5.1 (#1623)

Browse Source 
* CUTLASS 3.5.1

* updates, optimizations, fixes
This commit is contained in:
Vijay Thakkar
2024-07-29 08:46:24 -04:00
committed by GitHub
parent 56b46e2d13
commit be60a0b272
312 changed files with 19793 additions and 6775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/conv/device/conv2d_fprop_implicit_gemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32_sm89.cu
+7 -7
View File
@@ -60,7 +60,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc_
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -104,7 +104,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe5m2nhwc_fe4m3nhwc_fe4m3nhw
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -148,7 +148,7 @@ TEST(SM89_Device_Conv2d_Fprop_Analytic_ImplicitGemm_fe5m2nhwc_fe4m3nhwc_fe5m2nhw
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e5m2_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -192,7 +192,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -236,7 +236,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -280,7 +280,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
@@ -324,7 +324,7 @@ TEST(SM89_Device_Conv2d_Fprop_Optimized_ImplicitGemm_fe4m3nhwc_fe4mnhwc_fe4mnhwc
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = cutlass::float_e4m3_t;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;;
using ElementAccumulator = float;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
test/unit/conv/device/conv2d_fprop_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Conv2dFpropSM80TestResidaulBlock() {
static void Conv2dFpropSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Conv2dFpropSM80TestResidaulBlock() {
TEST(SM80_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Conv2dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_fprop_with_broadcast_sm70.cu
+2 -2
View File
@@ -60,7 +60,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = false
>
void TestResidaulBlock() {
void Conv2dFpropSM70TestResidualBlock() {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
@@ -117,7 +117,7 @@ void TestResidaulBlock() {
TEST(SM70_Device_Conv2d_Fprop_With_Residual_Block_Plus_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// Resnet
TestResidaulBlock<float, cutlass::epilogue::thread::ReLu, cutlass::plus, cutlass::epilogue::thread::Identity>();
Conv2dFpropSM70TestResidualBlock<float, cutlass::epilogue::thread::ReLu, cutlass::plus, cutlass::epilogue::thread::Identity>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_fprop_with_broadcast_sm75.cu
+3 -3
View File
@@ -103,7 +103,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
void Conv2dFpropSM75TestResidualBlock() {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
@@ -160,14 +160,14 @@ void TestResidaulBlock() {
TEST(SM75_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// Resnet
TestResidaulBlock<cutlass::half_t, cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM75TestResidualBlock<cutlass::half_t, cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
TEST(SM75_Device_Conv2d_Fprop_With_Residual_Block_Multiply_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_32x2_64x64x32) {
// EfficientNet V2
// Do not run split-K tests since the activation op is not Identity.
TestResidaulBlock<float, cutlass::epilogue::thread::Sigmoid, cutlass::multiplies, cutlass::epilogue::thread::Identity, false>();
Conv2dFpropSM75TestResidualBlock<float, cutlass::epilogue::thread::Sigmoid, cutlass::multiplies, cutlass::epilogue::thread::Identity, false>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_fprop_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Conv3dFpropSM80TestResidaulBlock() {
static void Conv3dFpropSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Conv3dFpropSM80TestResidaulBlock() {
TEST(SM80_Device_Conv3d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Conv3dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv3dFpropSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_with_broadcast_testbed.h
+37 -20
View File
@@ -204,17 +204,29 @@ public:
}
void initialize(
cutlass::conv::Conv3dProblemSize const &problem_size, uint64_t seed = 2019) {
cutlass::conv::Conv3dProblemSize const &problem_size, bool non_packed_test = false, uint64_t seed = 2019) {
tensor_A.resize(implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size));
tensor_B.resize(implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size));
tensor_C.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_C_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Z_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Z_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
// to make the layout of tensors a little bit bigger than the problem size
cutlass::Tensor5DCoord stride_increment = cutlass::Tensor5DCoord(8, 16, 32, 32, 64);
cutlass::Tensor5DCoord tensor_A_extent = implicit_gemm_tensor_a_extent(kConvolutionalOperator, problem_size);
cutlass::Tensor5DCoord tensor_B_extent = implicit_gemm_tensor_b_extent(kConvolutionalOperator, problem_size);
cutlass::Tensor5DCoord tensor_C_extent = implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size);
if (non_packed_test) {
tensor_A_extent += stride_increment;
tensor_C_extent += stride_increment;
}
tensor_A.resize(tensor_A_extent);
tensor_B.resize(tensor_B_extent);
tensor_C.resize(tensor_C_extent);
tensor_C_reference.resize(tensor_C_extent);
tensor_Z_computed.resize(tensor_C_extent);
tensor_Z_reference.resize(tensor_C_extent);
tensor_T_computed.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_T_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Y_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
tensor_Y_reference.resize(tensor_C_extent);
tensor_Broadcast.resize({
1,
1,
@@ -282,6 +294,7 @@ public:
bool run(
cutlass::conv::Conv3dProblemSize const &problem_size,
cutlass::conv::SplitKMode const &split_k_mode = cutlass::conv::SplitKMode::kSerial,
bool non_packed_test = false,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(1)) {
@@ -300,7 +313,7 @@ public:
<< std::endl;
#endif
initialize(problem_size);
initialize(problem_size, non_packed_test);
// configure the operator
Conv3d conv3d_op;
@@ -479,6 +492,7 @@ public:
<< problem_size.dilation_h << "x"
<< problem_size.dilation_w << "_"
<< (problem_size.mode == cutlass::conv::Mode::kCrossCorrelation ? "xcorr_" : "conv_")
<< (non_packed_test ? "non_packed_tensor_test_" : "packed_tensor_test_")
<< Conv3d::ThreadblockShape::kM << "x"
<< Conv3d::ThreadblockShape::kN << "x"
<< Conv3d::ThreadblockShape::kK << "_"
@@ -521,7 +535,8 @@ template <typename ImplicitGemm,
>
bool TestAllConv3dWithBroadcast(
const Conv3dProblemVector &conv_test_sizes = Conv3dProblemVector(),
const Conv3dProblemVector &conv_blacklist_sizes = Conv3dProblemVector()) {
const Conv3dProblemVector &conv_blacklist_sizes = Conv3dProblemVector(),
bool non_packed_test = false) {
bool passed = true;
@@ -595,17 +610,17 @@ bool TestAllConv3dWithBroadcast(
// test mode = xcross
passed = testbed.run(
conv_problem,
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
// test mode = convolution
passed = testbed.run(
conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
@@ -651,6 +666,7 @@ bool TestAllConv3dWithBroadcast(
passed = testbed.run(
conv3d_split_k_test_size.reset_split_k_slices(split_k_slice),
split_k_mode,
false,/*non_packed_test*/
cutlass::from_real<typename ImplicitGemm::ElementCompute>(alpha),
cutlass::from_real<typename ImplicitGemm::ElementCompute>(beta));
@@ -669,7 +685,8 @@ template <typename ImplicitGemm,
typename ReferenceOp = Conv3dWithBroadcastReferenceOp<ImplicitGemm>,
bool AddBroadcastFirst = false>
bool TestSpecificConv3dWithBroadcast(
const Conv3dProblemVector & problem_sizes) {
const Conv3dProblemVector & problem_sizes,
bool non_packed_test = false) {
bool passed = true;
@@ -686,19 +703,19 @@ bool TestSpecificConv3dWithBroadcast(
// Test
//
// test mode = xcross
// test mode = xcross, non_packed_test = false
passed = testbed.run(
conv_problem,
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
}
// test mode = convolution
// test mode = convolution, non_packed_test = false
passed = testbed.run(
conv_problem.reset_mode(cutlass::conv::Mode::kConvolution),
cutlass::conv::SplitKMode::kSerial);
cutlass::conv::SplitKMode::kSerial, non_packed_test);
if (!passed) {
return false;
test/unit/conv/device/deconv2d_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -104,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv2dSM80TestResidaulBlock() {
static void Deconv2dSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -163,7 +163,7 @@ static void Deconv2dSM80TestResidaulBlock() {
TEST(SM80_Device_Deconv2d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv2dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Deconv2dSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/deconv3d_with_broadcast_simt_sm80.cu
+2 -2
View File
@@ -103,7 +103,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv3dSM80TestResidaulBlock() {
static void Deconv3dSM80TestResidualBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -162,7 +162,7 @@ static void Deconv3dSM80TestResidaulBlock() {
TEST(SM80_Device_Deconv3d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv3dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Deconv3dSM80TestResidualBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////