leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

Updates for CUTLASS 3.5.0 (#1468)

Browse Source
This commit is contained in:
Vijay Thakkar
2024-04-11 21:33:40 -04:00
committed by GitHub
parent a40e08e9d5
commit 7d49e6c7e2
171 changed files with 7526 additions and 1888 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/conv/cache_testbed_output.h
+5 -8
View File
@@ -122,19 +122,15 @@ inline std::ostream &operator<<(std::ostream &out, CachedTestKey const &result)
struct CachedTestResult {
uint32_t D;
uint32_t sum;
uint32_t sum_of_square;
uint32_t second_sum_of_square;
//
// Methods
//
CachedTestResult(): D(), sum(), sum_of_square(), second_sum_of_square() { }
CachedTestResult(): D()
{ }
CachedTestResult(uint32_t D): D(D), sum(), sum_of_square(), second_sum_of_square() { }
CachedTestResult(uint32_t D, uint32_t sum, uint32_t sum_of_square, uint32_t second_sum_of_square):
D(D), sum(sum), sum_of_square(sum_of_square), second_sum_of_square(second_sum_of_square) { }
CachedTestResult(uint32_t D): D(D)
{ }
operator bool() const {
return bool(D);
@@ -262,6 +258,7 @@ inline char const *EncodeOperator(cutlass::conv::Operator conv_op) {
case cutlass::conv::Operator::kFprop: return "fprop";
case cutlass::conv::Operator::kDgrad: return "dgrad";
case cutlass::conv::Operator::kWgrad: return "wgrad";
case cutlass::conv::Operator::kDeconv: return "deconv";
}
return "conv_unknown";
}
test/unit/conv/device/CMakeLists.txt
+7
View File
@@ -140,14 +140,19 @@ if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
conv2d_fprop_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu
conv2d_dgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu
conv2d_wgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu
deconv2d_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu
conv2d_fprop_with_broadcast_simt_sm80.cu
deconv2d_with_broadcast_simt_sm80.cu
conv3d_fprop_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
conv3d_dgrad_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
conv3d_wgrad_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
deconv3d_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
conv3d_fprop_with_broadcast_simt_sm80.cu
deconv3d_with_broadcast_simt_sm80.cu
)
endif()
@@ -176,6 +181,7 @@ cutlass_test_unit_add_executable(
conv2d_fprop_with_broadcast_sm75.cu
conv2d_fprop_with_reduction_sm75.cu
conv3d_fprop_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm75.cu
conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm75.cu
)
@@ -209,6 +215,7 @@ if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
conv2d_strided_dgrad_implicit_gemm_swizzling4_sm80.cu
# Conv3d
conv3d_fprop_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
# Group Conv2d
test/unit/conv/device/conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
+3 -3
View File
@@ -85,7 +85,7 @@ TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tens
}
////////////////////////////////////////////////////////////////////////////////
#if 0
TEST(SM80_Device_Conv2d_Fprop_Precomputed_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_64x3_64x64x64) {
@@ -116,7 +116,8 @@ TEST(SM80_Device_Conv2d_Fprop_Precomputed_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_t
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
@@ -124,7 +125,6 @@ TEST(SM80_Device_Conv2d_Fprop_Precomputed_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_t
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
}
#endif
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
test/unit/conv/device/conv2d_fprop_with_broadcast_simt_sm80.cu
+4 -3
View File
@@ -81,7 +81,8 @@ TEST(SM80_Device_Conv2d_Fprop_With_Broadcast_Analytic_ImplicitGemm_f32nhwc_f32nh
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
@@ -103,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
static void Conv2dFpropSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -161,7 +162,7 @@ void TestResidaulBlock() {
TEST(SM80_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_testbed.h
+8 -8
View File
@@ -153,7 +153,6 @@ public:
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
@@ -489,7 +488,8 @@ public:
fname << "error_Conv2d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< ss_problem_size_text.str()
<< Conv2d::ThreadblockShape::kM << "x"
<< Conv2d::ThreadblockShape::kN << "x"
@@ -635,8 +635,8 @@ bool TestAllConv2d(
//
// CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity)) {
if (!((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -663,8 +663,8 @@ bool TestAllConv2d(
// CUTLASS DGRAD's *strided* stride specialization supports all stride {stride_h, stride_w}
// Although strided dgrad works for all stride combinations, we are only going
// to run strided dgrad for non-unity strides
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -718,8 +718,8 @@ bool TestAllConv2d(
}
// CUTLASS DGRAD's *strided* specialization does not support split-k mode
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
test/unit/conv/device/conv2d_with_broadcast_testbed.h
+11 -10
View File
@@ -404,9 +404,9 @@ public:
// compute tensor Z and tensor T
for (int n = 0; n < problem_size.N; ++n) {
for (int p = 0; p < problem_size.P; ++p) {
for (int q = 0; q < problem_size.Q; ++q) {
for (int k = 0; k < problem_size.K; ++k) {
for (int p = 0; p < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.P : problem_size.H); ++p) {
for (int q = 0; q < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.Q : problem_size.W); ++q) {
for (int k = 0; k < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.K : problem_size.C); ++k) {
ElementZ z{};
ElementT t{};
@@ -449,7 +449,8 @@ public:
fname << "error_Conv2d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< "nhwc_"
<< problem_size.N << "x"
<< problem_size.H << "x"
@@ -602,8 +603,8 @@ bool TestAllConv2dWithBroadcast(
//
// CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity)) {
if (!((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -613,8 +614,8 @@ bool TestAllConv2dWithBroadcast(
#if 0 // relax restrictions on analytic strided dgrad
// CUTLASS DGRAD's *strided* specialization only support stride >= {2, 2}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -650,8 +651,8 @@ bool TestAllConv2dWithBroadcast(
}
// CUTLASS DGRAD's *strided* specialization does not support split-k mode
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
test/unit/conv/device/conv3d_fprop_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
+2 -1
View File
@@ -111,7 +111,8 @@ TEST(SM80_Device_Conv3d_Fprop_Optimized_ImplicitGemm_f16ndhwc_f16ndhwc_f32ndhwc_
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv3dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv3dFpropKernel>;
test/unit/conv/device/conv3d_fprop_with_broadcast_simt_sm80.cu
+4 -3
View File
@@ -81,7 +81,8 @@ TEST(SM80_Device_Conv3d_Fprop_With_Broadcast_Analytic_ImplicitGemm_f32ndhwc_f32n
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv3dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv3dFpropKernel>;
@@ -103,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
static void Conv3dFpropSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -161,7 +162,7 @@ void TestResidaulBlock() {
TEST(SM80_Device_Conv3d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv3dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv3d_testbed.h
+11 -11
View File
@@ -169,7 +169,7 @@ public:
tensor_D_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
initialize_tensor(tensor_A.host_view(), init_A, seed);
initialize_tensor(tensor_B.host_view(), init_B, seed * 17);
initialize_tensor(tensor_B.host_view(), init_B, seed * 17);
initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
tensor_A.sync_device();
@@ -358,12 +358,12 @@ public:
bool cached_result_loaded = false;
CachedTestResult cached_test_result;
std::string conv2d_result_cache_name =
std::string conv3d_result_cache_name =
std::string("cached_results_") + CUTLASS_TARGET_NAME + ".txt";
if (CUTLASS_TEST_ENABLE_CACHED_RESULTS) {
CachedTestResultListing cached_results(conv2d_result_cache_name);
CachedTestResultListing cached_results(conv3d_result_cache_name);
auto cached = cached_results.find(cached_test_key);
@@ -376,7 +376,7 @@ public:
if (!cached_result_loaded) {
#if CUTLASS_CONV_TEST_UNIT_REFERENCE_DEVICE_ENABLED
cutlass::reference::device::Conv3d<
ElementA,
LayoutA,
@@ -426,15 +426,14 @@ public:
cached_test_result.D = TensorHash(tensor_D_reference.host_view());
CachedTestResultListing cached_results(conv2d_result_cache_name);
CachedTestResultListing cached_results(conv3d_result_cache_name);
cached_results.append(cached_test_key, cached_test_result);
cached_results.write(conv2d_result_cache_name);
cached_results.write(conv3d_result_cache_name);
}
} // if (!cached_result_loaded)
uint32_t tensor_D_hash = TensorHash(tensor_D_computed.host_view());
if (CUTLASS_TEST_ENABLE_CACHED_RESULTS) {
passed = (tensor_D_hash == cached_test_result.D);
@@ -456,7 +455,8 @@ public:
fname << "error_Conv3d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< "ndhwc_"
<< problem_size.N << "x"
<< problem_size.D << "x"
@@ -571,8 +571,8 @@ bool TestAllConv3d(
//
// CUTLASS DGRAD's unity stride specialization only support stride {1, 1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
((ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity) ||
(ImplicitGemm::UnderlyingKernel::Mma::IteratorB::kStrideSupport ==
test/unit/conv/device/conv3d_with_broadcast_testbed.h
+10 -11
View File
@@ -227,7 +227,6 @@ public:
initialize_tensor(tensor_B.host_view(), init_B, seed * 17);
initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
initialize_tensor(tensor_Broadcast.host_view(), init_C, seed * 39);
for (int n = 0; n < tensor_C_reference.extent().n(); ++n) {
for (int o = 0; o < tensor_C_reference.extent().d(); ++o) {
for (int p = 0; p < tensor_C_reference.extent().h(); ++p) {
@@ -239,7 +238,6 @@ public:
}
}
}
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
@@ -407,10 +405,10 @@ public:
// compute tensor Z and tensor T
for (int n = 0; n < problem_size.N; ++n) {
for (int o = 0; o < problem_size.Z; ++o) {
for (int p = 0; p < problem_size.P; ++p) {
for (int q = 0; q < problem_size.Q; ++q) {
for (int k = 0; k < problem_size.K; ++k) {
for (int o = 0; o < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.Z : problem_size.D); ++o) {
for (int p = 0; p < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.P : problem_size.H); ++p) {
for (int q = 0; q < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.Q : problem_size.W); ++q) {
for (int k = 0; k < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.K : problem_size.C); ++k) {
ElementZ z{};
ElementT t{};
@@ -454,7 +452,8 @@ public:
fname << "error_Conv3d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< "nnhwc_"
<< problem_size.N << "x"
<< problem_size.D << "x"
@@ -563,8 +562,8 @@ bool TestAllConv3dWithBroadcast(
//
// CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity)) {
if (!((conv_problem.stride_d == 1) &&
@@ -577,8 +576,8 @@ bool TestAllConv3dWithBroadcast(
#if 0 // relax restrictions on analytic strided dgrad
// CUTLASS DGRAD's *strided* specialization only support stride >= {2, 2}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
if (((conv_problem.stride_d == 1) && (conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
test/unit/conv/device/deconv2d_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu
+139
View File
@@ -0,0 +1,139 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_deconv2d.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv2d_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv2d instance
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2d<
ElementA,
cutlass::layout::TensorNHWC,
ElementB,
cutlass::layout::TensorNHWC,
ElementC,
cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Deconv2d>());
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv2d_Fprop_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_64x32x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv2d instance
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2d<
ElementA,
cutlass::layout::TensorNHWC,
ElementB,
cutlass::layout::TensorNHWC,
ElementC,
cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<64, 32, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Deconv2d>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
test/unit/conv/device/deconv2d_with_broadcast_simt_sm80.cu
+173
View File
@@ -0,0 +1,173 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/array.h"
#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
#include "cutlass/epilogue/thread/linear_combination_residual_block.h"
#include "cutlass/epilogue/thread/activation.h"
#include "cutlass/conv/kernel/default_deconv2d_with_broadcast.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_with_broadcast_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
TEST(SM80_Device_Deconv2d_With_Broadcast_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_32x2_64x64x32) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
ElementC,
ElementAccumulator,
ElementCompute,
ElementC,
ElementC,
1,
cutlass::epilogue::thread::ReLu<float>
>;
/// Device-level Conv2d instance
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2dWithBroadcast<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2dWithBroadcast<Deconv2d>());
}
// Test residual block fusion: UnaryOp(BinaryOp(ActivationOp(Conv2d(X) + bias), residual))
// LinearCombinationResidualBlock does not support the split-k mode unless ActivationOp is Identity.
// This is because the activation needs to be applied to the fully accumulated output of the Conv2d op,
// which only the last thread block would have an access to, before applying BinaryOp.
// The epilogue functor in the last thread block would have to be given three inputs, namely
// partial outputs, bias, and residual, but this is not supported in the current interface.
// Set TestSplitK = false to skip split-k tests with non-trivial ActivationOp.
template <
template<typename T> class ActivationOp,
template<typename T> class BinaryOp,
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv2dSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementD = ElementC;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationResidualBlock<
ElementD,
ElementAccumulator,
ElementCompute,
ElementC,
1,
ActivationOp,
BinaryOp,
UnaryOp
>;
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2dWithBroadcast<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
struct ReferenceOp {
using OutputOp = typename Deconv2d::EpilogueOutputOp;
using ElementZ = typename OutputOp::ElementZ;
ActivationOp<ElementCompute> activation;
BinaryOp<ElementCompute> binary_op;
UnaryOp<ElementCompute> unary_op;
void operator()(ElementZ &Z, ElementZ&, ElementCompute conv2d, ElementCompute residual) {
Z = ElementZ(unary_op(binary_op(activation(conv2d), residual)));
}
};
bool passed = test::conv::device::TestAllConv2dWithBroadcast<Deconv2d, ReferenceOp, true, TestSplitK>();
EXPECT_TRUE(passed);
}
TEST(SM80_Device_Deconv2d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv2dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/deconv3d_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
+141
View File
@@ -0,0 +1,141 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_deconv3d.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv3d_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv3d_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv3d instance
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3d<
ElementA,
cutlass::layout::TensorNDHWC,
ElementB,
cutlass::layout::TensorNDHWC,
ElementC,
cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
/// Run all unit test sizes with device-level Conv3d instance
EXPECT_TRUE(test::conv::device::TestAllConv3d<Deconv3d>());
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv3d_Optimized_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_64x32x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv3d instance
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3d<
ElementA,
cutlass::layout::TensorNDHWC,
ElementB,
cutlass::layout::TensorNDHWC,
ElementC,
cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<64, 32, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
/// Run all unit test sizes with device-level Conv3d instance
EXPECT_TRUE(test::conv::device::TestAllConv3d<Deconv3d>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
test/unit/conv/device/deconv3d_with_broadcast_simt_sm80.cu
+172
View File
@@ -0,0 +1,172 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/array.h"
#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
#include "cutlass/epilogue/thread/linear_combination_residual_block.h"
#include "cutlass/epilogue/thread/activation.h"
#include "cutlass/conv/kernel/default_deconv3d_with_broadcast.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv3d_with_broadcast_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
TEST(SM80_Device_Deconv3d_With_Broadcast_Optimized_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_32x2_64x64x32) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
ElementC,
ElementAccumulator,
ElementCompute,
ElementC,
ElementC,
1,
cutlass::epilogue::thread::ReLu<float>
>;
/// Device-level Conv3d instance
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3dWithBroadcast<
ElementA, cutlass::layout::TensorNDHWC,
ElementB, cutlass::layout::TensorNDHWC,
ElementC, cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
/// Run all unit test sizes with device-level Conv3d instance
EXPECT_TRUE(test::conv::device::TestAllConv3dWithBroadcast<Deconv3d>());
}
// Test residual block fusion: UnaryOp(BinaryOp(ActivationOp(Conv3d(X) + bias), residual))
// LinearCombinationResidualBlock does not support the split-k mode unless ActivationOp is Identity.
// This is because the activation needs to be applied to the fully accumulated output of the Conv3d op,
// which only the last thread block would have an access to, before applying BinaryOp.
// The epilogue functor in the last thread block would have to be given three inputs, namely
// partial outputs, bias, and residual, but this is not supported in the current interface.
// Set TestSplitK = false to skip split-k tests with non-trivial ActivationOp.
template <
template<typename T> class ActivationOp,
template<typename T> class BinaryOp,
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv3dSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementD = ElementC;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationResidualBlock<
ElementD,
ElementAccumulator,
ElementCompute,
ElementC,
1,
ActivationOp,
BinaryOp,
UnaryOp
>;
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3dWithBroadcast<
ElementA, cutlass::layout::TensorNDHWC,
ElementB, cutlass::layout::TensorNDHWC,
ElementC, cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
struct ReferenceOp {
using OutputOp = typename Deconv3d::EpilogueOutputOp;
using ElementZ = typename OutputOp::ElementZ;
ActivationOp<ElementCompute> activation;
BinaryOp<ElementCompute> binary_op;
UnaryOp<ElementCompute> unary_op;
void operator()(ElementZ &Z, ElementZ&, ElementCompute conv3d, ElementCompute residual) {
Z = ElementZ(unary_op(binary_op(activation(conv3d), residual)));
}
};
bool passed = test::conv::device::TestAllConv3dWithBroadcast<Deconv3d, ReferenceOp, true, TestSplitK>();
EXPECT_TRUE(passed);
}
TEST(SM80_Device_Deconv3d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv3dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device_3x/CMakeLists.txt
+1
View File
@@ -26,6 +26,7 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add_subdirectory(fprop)
add_subdirectory(wgrad)
add_subdirectory(dgrad)
test/unit/conv/device_3x/testbed_conv.hpp
+9 -15
View File
@@ -53,7 +53,6 @@
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/reference/device/tensor_fill.h"
#include "cutlass/util/reference/device/tensor_compare.h"
#include "conv_problem_sizes.hpp"
#include "../cache_testbed_output.h"
@@ -195,7 +194,8 @@ struct ConvTestbed {
bool run(
ProblemShape const& problem_shape,
ElementScalar alpha = ElementScalar(1),
ElementScalar beta = ElementScalar(0)) {
ElementScalar beta = ElementScalar(0)
) {
// Waive test if insufficient CUDA device
if (!sufficient()) {
@@ -250,14 +250,16 @@ struct ConvTestbed {
auto &fusion_args = args.epilogue.thread;
// some fused patterns have no linear combination
fusion_args.alpha = alpha;
fusion_args.beta = beta;
if constexpr (IsBiasEnabled) {
fusion_args.bias_ptr = tensor_bias.data().get();
}
// Clamp bound
if constexpr (cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>) {
fusion_args.activation.lower_bound = ElementCompute{0};
fusion_args.activation.lower_bound = CUTLASS_STL_NAMESPACE::numeric_limits<ElementCompute>::lowest();
fusion_args.activation.upper_bound = CUTLASS_STL_NAMESPACE::numeric_limits<ElementCompute>::max();
}
@@ -422,17 +424,11 @@ struct ConvTestbed {
reference_impl.compute_reference();
}
// Validate kernel against reference
passed = compare_reference(
mD_ref, mD_computed, mA, mB, mAlpha,
mBeta, mBias,
this->epsilon);
passed = compare_reference(mD_ref, mD_computed, mA, mB, mAlpha, mBeta, mBias, this->epsilon);
}
#else
// Validate kernel against reference
passed = compare_reference(
mD_ref, mD_computed, mA, mB, mAlpha,
mBeta, mBias,
this->epsilon);
passed = compare_reference(mD_ref, mD_computed, mA, mB, mAlpha, mBeta, mBias, this->epsilon);
#endif
EXPECT_TRUE(passed);
@@ -445,8 +441,7 @@ struct ConvTestbed {
class EngineB, class LayoutB,
class EngineAlpha, class LayoutAlpha,
class EngineBeta, class LayoutBeta,
class EngineBias, class LayoutBias
>
class EngineBias, class LayoutBias>
static constexpr bool
compare_reference(
cute::Tensor<Engine, Layout> const& reference,
@@ -503,7 +498,6 @@ struct ConvTestbed {
printf("[%ld]: bias = %f\n", i, float(tensor_bias(i)));
}
}
for (size_t i = 0; i < size_t(size(reference)); ++i) {
printf("[%ld]: ref = %f, computed = %f\n", i, float(reference(i)), float(computed(i)));
}