leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Updates for CUTLASS 3.5.0 (#1468)

Browse Source
This commit is contained in:
Vijay Thakkar
2024-04-11 21:33:40 -04:00
committed by GitHub
parent a40e08e9d5
commit 7d49e6c7e2
171 changed files with 7526 additions and 1888 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
test/unit/conv/cache_testbed_output.h
View File

@@ -122,19 +122,15 @@ inline std::ostream &operator<<(std::ostream &out, CachedTestKey const &result)
struct CachedTestResult {
uint32_t D;
uint32_t sum;
uint32_t sum_of_square;
uint32_t second_sum_of_square;
//
// Methods
//
CachedTestResult(): D(), sum(), sum_of_square(), second_sum_of_square() { }
CachedTestResult(): D()
{ }
CachedTestResult(uint32_t D): D(D), sum(), sum_of_square(), second_sum_of_square() { }
CachedTestResult(uint32_t D, uint32_t sum, uint32_t sum_of_square, uint32_t second_sum_of_square):
D(D), sum(sum), sum_of_square(sum_of_square), second_sum_of_square(second_sum_of_square) { }
CachedTestResult(uint32_t D): D(D)
{ }
operator bool() const {
return bool(D);
@@ -262,6 +258,7 @@ inline char const *EncodeOperator(cutlass::conv::Operator conv_op) {
case cutlass::conv::Operator::kFprop: return "fprop";
case cutlass::conv::Operator::kDgrad: return "dgrad";
case cutlass::conv::Operator::kWgrad: return "wgrad";
case cutlass::conv::Operator::kDeconv: return "deconv";
}
return "conv_unknown";
}

7
test/unit/conv/device/CMakeLists.txt
View File

@@ -140,14 +140,19 @@ if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
conv2d_fprop_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu
conv2d_dgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu
conv2d_wgrad_implicit_gemm_cf32nhwc_cf32nhwc_cf32nhwc_simt_f32_sm80.cu
deconv2d_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu
conv2d_fprop_with_broadcast_simt_sm80.cu
deconv2d_with_broadcast_simt_sm80.cu
conv3d_fprop_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
conv3d_dgrad_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
conv3d_wgrad_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
deconv3d_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu
conv3d_fprop_with_broadcast_simt_sm80.cu
deconv3d_with_broadcast_simt_sm80.cu
)
endif()
@@ -176,6 +181,7 @@ cutlass_test_unit_add_executable(
conv2d_fprop_with_broadcast_sm75.cu
conv2d_fprop_with_reduction_sm75.cu
conv3d_fprop_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm75.cu
conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm75.cu
)
@@ -209,6 +215,7 @@ if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
conv2d_strided_dgrad_implicit_gemm_swizzling4_sm80.cu
# Conv3d
conv3d_fprop_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
# Group Conv2d

6
test/unit/conv/device/conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
View File

@@ -85,7 +85,7 @@ TEST(SM80_Device_Conv2d_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tens
}
////////////////////////////////////////////////////////////////////////////////
#if 0
TEST(SM80_Device_Conv2d_Fprop_Precomputed_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32,
128x128_64x3_64x64x64) {
@@ -116,7 +116,8 @@ TEST(SM80_Device_Conv2d_Fprop_Precomputed_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_t
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
@@ -124,7 +125,6 @@ TEST(SM80_Device_Conv2d_Fprop_Precomputed_ImplicitGemm_f16nhwc_f16nhwc_f32nhwc_t
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dFprop>());
}
#endif
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED

7
test/unit/conv/device/conv2d_fprop_with_broadcast_simt_sm80.cu
View File

@@ -81,7 +81,8 @@ TEST(SM80_Device_Conv2d_Fprop_With_Broadcast_Analytic_ImplicitGemm_f32nhwc_f32nh
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv2dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dFpropKernel>;
@@ -103,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
static void Conv2dFpropSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -161,7 +162,7 @@ void TestResidaulBlock() {
TEST(SM80_Device_Conv2d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv2dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////

16
test/unit/conv/device/conv2d_testbed.h
View File

@@ -153,7 +153,6 @@ public:
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
@@ -489,7 +488,8 @@ public:
fname << "error_Conv2d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< ss_problem_size_text.str()
<< Conv2d::ThreadblockShape::kM << "x"
<< Conv2d::ThreadblockShape::kN << "x"
@@ -635,8 +635,8 @@ bool TestAllConv2d(
//
// CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity)) {
if (!((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -663,8 +663,8 @@ bool TestAllConv2d(
// CUTLASS DGRAD's *strided* stride specialization supports all stride {stride_h, stride_w}
// Although strided dgrad works for all stride combinations, we are only going
// to run strided dgrad for non-unity strides
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -718,8 +718,8 @@ bool TestAllConv2d(
}
// CUTLASS DGRAD's *strided* specialization does not support split-k mode
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {

21
test/unit/conv/device/conv2d_with_broadcast_testbed.h
View File

@@ -404,9 +404,9 @@ public:
// compute tensor Z and tensor T
for (int n = 0; n < problem_size.N; ++n) {
for (int p = 0; p < problem_size.P; ++p) {
for (int q = 0; q < problem_size.Q; ++q) {
for (int k = 0; k < problem_size.K; ++k) {
for (int p = 0; p < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.P : problem_size.H); ++p) {
for (int q = 0; q < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.Q : problem_size.W); ++q) {
for (int k = 0; k < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.K : problem_size.C); ++k) {
ElementZ z{};
ElementT t{};
@@ -449,7 +449,8 @@ public:
fname << "error_Conv2d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv2d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< "nhwc_"
<< problem_size.N << "x"
<< problem_size.H << "x"
@@ -602,8 +603,8 @@ bool TestAllConv2dWithBroadcast(
//
// CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity)) {
if (!((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -613,8 +614,8 @@ bool TestAllConv2dWithBroadcast(
#if 0 // relax restrictions on analytic strided dgrad
// CUTLASS DGRAD's *strided* specialization only support stride >= {2, 2}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
if (((conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {
@@ -650,8 +651,8 @@ bool TestAllConv2dWithBroadcast(
}
// CUTLASS DGRAD's *strided* specialization does not support split-k mode
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {

3
test/unit/conv/device/conv3d_fprop_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
View File

@@ -111,7 +111,8 @@ TEST(SM80_Device_Conv3d_Fprop_Optimized_ImplicitGemm_f16ndhwc_f16ndhwc_f32ndhwc_
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv3dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv3dFpropKernel>;

7
test/unit/conv/device/conv3d_fprop_with_broadcast_simt_sm80.cu
View File

@@ -81,7 +81,8 @@ TEST(SM80_Device_Conv3d_Fprop_With_Broadcast_Analytic_ImplicitGemm_f32ndhwc_f32n
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Conv3dFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv3dFpropKernel>;
@@ -103,7 +104,7 @@ template <
template<typename T> class UnaryOp,
bool TestSplitK = true
>
void TestResidaulBlock() {
static void Conv3dFpropSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
@@ -161,7 +162,7 @@ void TestResidaulBlock() {
TEST(SM80_Device_Conv3d_Fprop_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
Conv3dFpropSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////

22
test/unit/conv/device/conv3d_testbed.h
View File

@@ -169,7 +169,7 @@ public:
tensor_D_reference.resize(implicit_gemm_tensor_c_extent(kConvolutionalOperator, problem_size));
initialize_tensor(tensor_A.host_view(), init_A, seed);
initialize_tensor(tensor_B.host_view(), init_B, seed * 17);
initialize_tensor(tensor_B.host_view(), init_B, seed * 17);
initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
tensor_A.sync_device();
@@ -358,12 +358,12 @@ public:
bool cached_result_loaded = false;
CachedTestResult cached_test_result;
std::string conv2d_result_cache_name =
std::string conv3d_result_cache_name =
std::string("cached_results_") + CUTLASS_TARGET_NAME + ".txt";
if (CUTLASS_TEST_ENABLE_CACHED_RESULTS) {
CachedTestResultListing cached_results(conv2d_result_cache_name);
CachedTestResultListing cached_results(conv3d_result_cache_name);
auto cached = cached_results.find(cached_test_key);
@@ -376,7 +376,7 @@ public:
if (!cached_result_loaded) {
#if CUTLASS_CONV_TEST_UNIT_REFERENCE_DEVICE_ENABLED
cutlass::reference::device::Conv3d<
ElementA,
LayoutA,
@@ -426,15 +426,14 @@ public:
cached_test_result.D = TensorHash(tensor_D_reference.host_view());
CachedTestResultListing cached_results(conv2d_result_cache_name);
CachedTestResultListing cached_results(conv3d_result_cache_name);
cached_results.append(cached_test_key, cached_test_result);
cached_results.write(conv2d_result_cache_name);
cached_results.write(conv3d_result_cache_name);
}
} // if (!cached_result_loaded)
uint32_t tensor_D_hash = TensorHash(tensor_D_computed.host_view());
if (CUTLASS_TEST_ENABLE_CACHED_RESULTS) {
passed = (tensor_D_hash == cached_test_result.D);
@@ -456,7 +455,8 @@ public:
fname << "error_Conv3d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< "ndhwc_"
<< problem_size.N << "x"
<< problem_size.D << "x"
@@ -571,8 +571,8 @@ bool TestAllConv3d(
//
// CUTLASS DGRAD's unity stride specialization only support stride {1, 1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
((ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity) ||
(ImplicitGemm::UnderlyingKernel::Mma::IteratorB::kStrideSupport ==

21
test/unit/conv/device/conv3d_with_broadcast_testbed.h
View File

@@ -227,7 +227,6 @@ public:
initialize_tensor(tensor_B.host_view(), init_B, seed * 17);
initialize_tensor(tensor_C.host_view(), init_C, seed * 39);
initialize_tensor(tensor_Broadcast.host_view(), init_C, seed * 39);
for (int n = 0; n < tensor_C_reference.extent().n(); ++n) {
for (int o = 0; o < tensor_C_reference.extent().d(); ++o) {
for (int p = 0; p < tensor_C_reference.extent().h(); ++p) {
@@ -239,7 +238,6 @@ public:
}
}
}
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
@@ -407,10 +405,10 @@ public:
// compute tensor Z and tensor T
for (int n = 0; n < problem_size.N; ++n) {
for (int o = 0; o < problem_size.Z; ++o) {
for (int p = 0; p < problem_size.P; ++p) {
for (int q = 0; q < problem_size.Q; ++q) {
for (int k = 0; k < problem_size.K; ++k) {
for (int o = 0; o < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.Z : problem_size.D); ++o) {
for (int p = 0; p < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.P : problem_size.H); ++p) {
for (int q = 0; q < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.Q : problem_size.W); ++q) {
for (int k = 0; k < (kConvolutionalOperator == cutlass::conv::Operator::kFprop ? problem_size.K : problem_size.C); ++k) {
ElementZ z{};
ElementT t{};
@@ -454,7 +452,8 @@ public:
fname << "error_Conv3d_ImplicitGemm_device_"
<< (split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial_reduction_" : "parallel_reduction_")
<< (Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kFprop ? "fprop_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" : "wgrad_"))
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ? "dgrad_" :
(Conv3d::kConvolutionalOperator == cutlass::conv::Operator::kDeconv ? "deconv_" : "wgrad_")))
<< "nnhwc_"
<< problem_size.N << "x"
<< problem_size.D << "x"
@@ -563,8 +562,8 @@ bool TestAllConv3dWithBroadcast(
//
// CUTLASS DGRAD's *unity* stride specialization only support stride {1, 1}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kUnity)) {
if (!((conv_problem.stride_d == 1) &&
@@ -577,8 +576,8 @@ bool TestAllConv3dWithBroadcast(
#if 0 // relax restrictions on analytic strided dgrad
// CUTLASS DGRAD's *strided* specialization only support stride >= {2, 2}
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&
if ((ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDgrad ||
ImplicitGemm::kConvolutionalOperator == cutlass::conv::Operator::kDeconv) &&
(ImplicitGemm::UnderlyingKernel::Mma::IteratorA::kStrideSupport ==
cutlass::conv::StrideSupport::kStrided)) {
if (((conv_problem.stride_d == 1) && (conv_problem.stride_h == 1) && (conv_problem.stride_w == 1))) {

139
test/unit/conv/device/deconv2d_implicit_gemm_f32nhwc_f32nhwc_f32nhwc_simt_f32_sm80.cu Normal file
View File

@@ -0,0 +1,139 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_deconv2d.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv2d_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv2d instance
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2d<
ElementA,
cutlass::layout::TensorNHWC,
ElementB,
cutlass::layout::TensorNHWC,
ElementC,
cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Deconv2d>());
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv2d_Fprop_Optimized_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_64x32x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv2d instance
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2d<
ElementA,
cutlass::layout::TensorNHWC,
ElementB,
cutlass::layout::TensorNHWC,
ElementC,
cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<64, 32, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Deconv2d>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED

173
test/unit/conv/device/deconv2d_with_broadcast_simt_sm80.cu Normal file
View File

@@ -0,0 +1,173 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/array.h"
#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
#include "cutlass/epilogue/thread/linear_combination_residual_block.h"
#include "cutlass/epilogue/thread/activation.h"
#include "cutlass/conv/kernel/default_deconv2d_with_broadcast.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_with_broadcast_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
TEST(SM80_Device_Deconv2d_With_Broadcast_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_32x2_64x64x32) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
ElementC,
ElementAccumulator,
ElementCompute,
ElementC,
ElementC,
1,
cutlass::epilogue::thread::ReLu<float>
>;
/// Device-level Conv2d instance
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2dWithBroadcast<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2dWithBroadcast<Deconv2d>());
}
// Test residual block fusion: UnaryOp(BinaryOp(ActivationOp(Conv2d(X) + bias), residual))
// LinearCombinationResidualBlock does not support the split-k mode unless ActivationOp is Identity.
// This is because the activation needs to be applied to the fully accumulated output of the Conv2d op,
// which only the last thread block would have an access to, before applying BinaryOp.
// The epilogue functor in the last thread block would have to be given three inputs, namely
// partial outputs, bias, and residual, but this is not supported in the current interface.
// Set TestSplitK = false to skip split-k tests with non-trivial ActivationOp.
template <
template<typename T> class ActivationOp,
template<typename T> class BinaryOp,
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv2dSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementD = ElementC;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationResidualBlock<
ElementD,
ElementAccumulator,
ElementCompute,
ElementC,
1,
ActivationOp,
BinaryOp,
UnaryOp
>;
using Deconv2dKernel = typename cutlass::conv::kernel::DefaultDeconv2dWithBroadcast<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv2d = cutlass::conv::device::ImplicitGemmConvolution<Deconv2dKernel>;
struct ReferenceOp {
using OutputOp = typename Deconv2d::EpilogueOutputOp;
using ElementZ = typename OutputOp::ElementZ;
ActivationOp<ElementCompute> activation;
BinaryOp<ElementCompute> binary_op;
UnaryOp<ElementCompute> unary_op;
void operator()(ElementZ &Z, ElementZ&, ElementCompute conv2d, ElementCompute residual) {
Z = ElementZ(unary_op(binary_op(activation(conv2d), residual)));
}
};
bool passed = test::conv::device::TestAllConv2dWithBroadcast<Deconv2d, ReferenceOp, true, TestSplitK>();
EXPECT_TRUE(passed);
}
TEST(SM80_Device_Deconv2d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32nhwc_f32nhwc_f32nhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv2dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
////////////////////////////////////////////////////////////////////////////////

141
test/unit/conv/device/deconv3d_implicit_gemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32_sm80.cu Normal file
View File

@@ -0,0 +1,141 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_deconv3d.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv3d_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv3d_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv3d instance
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3d<
ElementA,
cutlass::layout::TensorNDHWC,
ElementB,
cutlass::layout::TensorNDHWC,
ElementC,
cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
/// Run all unit test sizes with device-level Conv3d instance
EXPECT_TRUE(test::conv::device::TestAllConv3d<Deconv3d>());
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Deconv3d_Optimized_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_64x32x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv3d instance
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3d<
ElementA,
cutlass::layout::TensorNDHWC,
ElementB,
cutlass::layout::TensorNDHWC,
ElementC,
cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<64, 32, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
/// Run all unit test sizes with device-level Conv3d instance
EXPECT_TRUE(test::conv::device::TestAllConv3d<Deconv3d>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED

172
test/unit/conv/device/deconv3d_with_broadcast_simt_sm80.cu Normal file
View File

@@ -0,0 +1,172 @@
/***************************************************************************************************
* Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/array.h"
#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
#include "cutlass/epilogue/thread/linear_combination_residual_block.h"
#include "cutlass/epilogue/thread/activation.h"
#include "cutlass/conv/kernel/default_deconv3d_with_broadcast.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv3d_with_broadcast_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
TEST(SM80_Device_Deconv3d_With_Broadcast_Optimized_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_32x2_64x64x32) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
ElementC,
ElementAccumulator,
ElementCompute,
ElementC,
ElementC,
1,
cutlass::epilogue::thread::ReLu<float>
>;
/// Device-level Conv3d instance
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3dWithBroadcast<
ElementA, cutlass::layout::TensorNDHWC,
ElementB, cutlass::layout::TensorNDHWC,
ElementC, cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
/// Run all unit test sizes with device-level Conv3d instance
EXPECT_TRUE(test::conv::device::TestAllConv3dWithBroadcast<Deconv3d>());
}
// Test residual block fusion: UnaryOp(BinaryOp(ActivationOp(Conv3d(X) + bias), residual))
// LinearCombinationResidualBlock does not support the split-k mode unless ActivationOp is Identity.
// This is because the activation needs to be applied to the fully accumulated output of the Conv3d op,
// which only the last thread block would have an access to, before applying BinaryOp.
// The epilogue functor in the last thread block would have to be given three inputs, namely
// partial outputs, bias, and residual, but this is not supported in the current interface.
// Set TestSplitK = false to skip split-k tests with non-trivial ActivationOp.
template <
template<typename T> class ActivationOp,
template<typename T> class BinaryOp,
template<typename T> class UnaryOp,
bool TestSplitK = true
>
static void Deconv3dSM80TestResidaulBlock() {
using ElementA = float;
using ElementB = float;
using ElementC = float;
using ElementD = ElementC;
using ElementCompute = float;
using ElementAccumulator = float;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationResidualBlock<
ElementD,
ElementAccumulator,
ElementCompute,
ElementC,
1,
ActivationOp,
BinaryOp,
UnaryOp
>;
using Deconv3dKernel = typename cutlass::conv::kernel::DefaultDeconv3dWithBroadcast<
ElementA, cutlass::layout::TensorNDHWC,
ElementB, cutlass::layout::TensorNDHWC,
ElementC, cutlass::layout::TensorNDHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<32, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
EpilogueOutputOp,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
4,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kOptimized,
cutlass::conv::StrideSupport::kUnity
>::Kernel;
using Deconv3d = cutlass::conv::device::ImplicitGemmConvolution<Deconv3dKernel>;
struct ReferenceOp {
using OutputOp = typename Deconv3d::EpilogueOutputOp;
using ElementZ = typename OutputOp::ElementZ;
ActivationOp<ElementCompute> activation;
BinaryOp<ElementCompute> binary_op;
UnaryOp<ElementCompute> unary_op;
void operator()(ElementZ &Z, ElementZ&, ElementCompute conv3d, ElementCompute residual) {
Z = ElementZ(unary_op(binary_op(activation(conv3d), residual)));
}
};
bool passed = test::conv::device::TestAllConv3dWithBroadcast<Deconv3d, ReferenceOp, true, TestSplitK>();
EXPECT_TRUE(passed);
}
TEST(SM80_Device_Deconv3d_With_Residual_Block_Plus_Analytic_ImplicitGemm_f32ndhwc_f32ndhwc_f32ndhwc_simt_f32,
128x128_8x4_32x64x8) {
// Resnet
Deconv3dSM80TestResidaulBlock<cutlass::epilogue::thread::Identity, cutlass::plus, cutlass::epilogue::thread::ReLu>();
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
////////////////////////////////////////////////////////////////////////////////

1
test/unit/conv/device_3x/CMakeLists.txt
View File

@@ -26,6 +26,7 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add_subdirectory(fprop)
add_subdirectory(wgrad)
add_subdirectory(dgrad)

24
test/unit/conv/device_3x/testbed_conv.hpp
View File

@@ -53,7 +53,6 @@
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/reference/device/tensor_fill.h"
#include "cutlass/util/reference/device/tensor_compare.h"
#include "conv_problem_sizes.hpp"
#include "../cache_testbed_output.h"
@@ -195,7 +194,8 @@ struct ConvTestbed {
bool run(
ProblemShape const& problem_shape,
ElementScalar alpha = ElementScalar(1),
ElementScalar beta = ElementScalar(0)) {
ElementScalar beta = ElementScalar(0)
) {
// Waive test if insufficient CUDA device
if (!sufficient()) {
@@ -250,14 +250,16 @@ struct ConvTestbed {
auto &fusion_args = args.epilogue.thread;
// some fused patterns have no linear combination
fusion_args.alpha = alpha;
fusion_args.beta = beta;
if constexpr (IsBiasEnabled) {
fusion_args.bias_ptr = tensor_bias.data().get();
}
// Clamp bound
if constexpr (cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>) {
fusion_args.activation.lower_bound = ElementCompute{0};
fusion_args.activation.lower_bound = CUTLASS_STL_NAMESPACE::numeric_limits<ElementCompute>::lowest();
fusion_args.activation.upper_bound = CUTLASS_STL_NAMESPACE::numeric_limits<ElementCompute>::max();
}
@@ -422,17 +424,11 @@ struct ConvTestbed {
reference_impl.compute_reference();
}
// Validate kernel against reference
passed = compare_reference(
mD_ref, mD_computed, mA, mB, mAlpha,
mBeta, mBias,
this->epsilon);
passed = compare_reference(mD_ref, mD_computed, mA, mB, mAlpha, mBeta, mBias, this->epsilon);
}
#else
// Validate kernel against reference
passed = compare_reference(
mD_ref, mD_computed, mA, mB, mAlpha,
mBeta, mBias,
this->epsilon);
passed = compare_reference(mD_ref, mD_computed, mA, mB, mAlpha, mBeta, mBias, this->epsilon);
#endif
EXPECT_TRUE(passed);
@@ -445,8 +441,7 @@ struct ConvTestbed {
class EngineB, class LayoutB,
class EngineAlpha, class LayoutAlpha,
class EngineBeta, class LayoutBeta,
class EngineBias, class LayoutBias
>
class EngineBias, class LayoutBias>
static constexpr bool
compare_reference(
cute::Tensor<Engine, Layout> const& reference,
@@ -503,7 +498,6 @@ struct ConvTestbed {
printf("[%ld]: bias = %f\n", i, float(tensor_bias(i)));
}
}
for (size_t i = 0; i < size_t(size(reference)); ++i) {
printf("[%ld]: ref = %f, computed = %f\n", i, float(reference(i)), float(computed(i)));
}

27
test/unit/core/CMakeLists.txt
View File

@@ -45,30 +45,3 @@ cutlass_test_unit_add_executable(
fast_numeric_conversion.cu
functional.cu
)
#
# CUTLASS 3x increases the host compiler requirements to C++17. However, there are
# certain existing integrations that will benefit from maintaining C++11 compatibility.
#
# This requirement only applies to select .h files which are explicitly annotated. It
# does not apply to any .hpp file.
#
# `cutlass_test_unit_core_cpp11` enforces the C++11 requirement.
#
set(CMAKE_CUDA_STANDARD 11)
set(CMAKE_CUDA_STANDARD_REQUIRED ON)
add_executable(
cutlass_test_unit_core_cpp11
cpp11.cu
)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
target_compile_options(
cutlass_test_unit_core_cpp11
PRIVATE
$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler -Werror>
)
endif()

87
test/unit/core/cpp11.cu
View File

@@ -1,87 +0,0 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*
Note: CUTLASS 3x increases the host compiler requirements to C++17. However, certain
existing integrations of CUTLASS require C++11 host compilers.
Until this requirement can be lifted, certain headers with this annotation are required
to be remain consistent with C++11 syntax.
C++11 compatibility is enforced by this unit test: `cutlass_test_unit_core_cpp11`.
*/
/////////////////////////////////////////////////////////////////////////////////////////////////
#include <cutlass/cutlass.h>
#include <cutlass/complex.h>
#include <cutlass/coord.h>
#include <cutlass/core_io.h>
#include <cutlass/array.h>
#include <cutlass/bfloat16.h>
#include <cutlass/fast_math.h>
#include <cutlass/float8.h>
#include <cutlass/half.h>
#include <cutlass/integer_subbyte.h>
#include <cutlass/kernel_hardware_info.h>
#include <cutlass/numeric_conversion.h>
#include <cutlass/numeric_size.h>
#include <cutlass/numeric_types.h>
#include <cutlass/tfloat32.h>
#include <cutlass/workspace.h>
#include <cutlass/subbyte_reference.h>
#include <cutlass/conv/convolution.h>
#include <cutlass/conv/conv2d_problem_size.h>
#include <cutlass/epilogue/threadblock/predicated_tile_iterator_params.h>
#include <cutlass/gemm/gemm_enumerated_types.h>
#include <cutlass/gemm/kernel/tile_scheduler_params.h>
#include <cutlass/gemm/threadblock/threadblock_swizzle_streamk.h>
#include <cutlass/layout/matrix.h>
#include <cutlass/layout/pitch_linear.h>
#include <cutlass/transform/threadblock/predicated_tile_access_iterator_params.h>
/////////////////////////////////////////////////////////////////////////////////////////////////
#if (201700L <= __cplusplus )
#error "This file and all of its includes must be compilable as C++11."
#endif
/////////////////////////////////////////////////////////////////////////////////////////////////
int main() {
return 0;
}
/////////////////////////////////////////////////////////////////////////////////////////////////

16
test/unit/core/numeric_conversion.cu
View File

@@ -108,8 +108,8 @@ __global__ void convert_with_scale_factor(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Destination, typename Source, typename ScaleFactor, int Count, int Range = 4>
void run_test_with_scalefactor(const char dest_name[], const char source_name[], const char scale_factor_name[]) {
template <typename Destination, typename Source, typename ScaleFactor, int Count>
void run_test_with_scalefactor(const char dest_name[], const char source_name[], const char scale_factor_name[], const int range = 4, const int offset = 0) {
const int kN = Count;
dim3 grid(1, 1);
@@ -124,7 +124,7 @@ void run_test_with_scalefactor(const char dest_name[], const char source_name[],
for (int i = 0; i < kN; ++i) {
source_ref.at({0, i}) = Source(i % Range);
source_ref.at({0, i}) = Source(i % range + offset);
}
for (int i = 0; i < kN; ++i) {
@@ -144,10 +144,12 @@ void run_test_with_scalefactor(const char dest_name[], const char source_name[],
for (int i = 0; i < kN; ++i) {
float ref = float(source_ref.at({0, i})) / float(scale_factor_ref.at({0, i}));
EXPECT_TRUE(float(destination_ref.at({0, i})) == ref)
<< "Destination type: " << dest_name << " "<< float(destination_ref.at({0, i}))
<< ", Source type: " << source_name << " " << float(source_ref.at({0, i}))
<< ", Count: " << Count;
bool pass = float(destination_ref.at({0, i})) == ref;
EXPECT_TRUE(pass)
<< "Destination type: " << dest_name << " "<< float(destination_ref.at({0, i})) << std::endl
<< ", Source type: " << source_name << " " << float(source_ref.at({0, i})) << std::endl
<< ", Scalefactor type: " << source_name << " " << float(scale_factor_ref.at({0, i})) << std::endl
<< ", idx: " << i << std::endl;
}
}

3
test/unit/cute/CMakeLists.txt
View File

@@ -28,6 +28,7 @@
add_subdirectory(core)
add_subdirectory(volta)
add_subdirectory(turing)
add_subdirectory(ampere)
add_subdirectory(hopper)
add_subdirectory(layout)
@@ -39,6 +40,7 @@ add_custom_target(
cutlass_test_unit_cute_layout
cutlass_test_unit_cute_core
cutlass_test_unit_cute_volta
cutlass_test_unit_cute_turing
cutlass_test_unit_cute_ampere
cutlass_test_unit_cute_hopper
cutlass_test_unit_cute_msvc_compilation
@@ -51,6 +53,7 @@ add_custom_target(
test_unit_cute_core
test_unit_cute_volta
test_unit_cute_ampere
test_unit_cute_turing
test_unit_cute_hopper
test_unit_cute_msvc_compilation
)

1
test/unit/cute/ampere/CMakeLists.txt
View File

@@ -30,6 +30,7 @@ cutlass_test_unit_add_executable(
cutlass_test_unit_cute_ampere
cp_async.cu
ldsm.cu
cooperative_gemm.cu
)
cutlass_test_unit_add_executable(

300
test/unit/cute/ampere/cooperative_gemm.cu Normal file
View File

@@ -0,0 +1,300 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include "../cooperative_gemm_common.hpp"
using namespace cute;
TEST(SM80_CuTe_Ampere, CooperativeGemm1_Half_MMA) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 64;
constexpr uint32_t n = 64;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x8_F16F16F16F16_TN>,
Layout<Shape<_2, _2, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm2_Double_MMA) {
using value_type = double;
constexpr uint32_t m = 64;
constexpr uint32_t n = 64;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_8x8x4_F64F64F64F64_TN>,
Layout<Shape<_2,_2,_1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm3_Half_MMA_CustomSmemLayouts) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 128;
constexpr uint32_t n = 128;
constexpr uint32_t k = 128;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x16_F16F16F16F16_TN>,
Layout<Shape<_2, _2, _1>>, // 2x2x1 thread group
Tile<_32, _32, _16> // 32x32x16 MMA for LDSM, 1x2x1 value group`
>;
using smem_a_atom_layout_t = Layout<Shape<_64, _8>, Stride< _1,_64>>;
using smem_b_atom_layout_t = Layout<Shape< _8,_32>, Stride<_32, _1>>;
using smem_c_atom_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{})));
test_cooperative_gemm_col_major_layout<smem_a_atom_layout_t,
smem_b_atom_layout_t,
smem_c_atom_layout_t,
m,
n,
k,
thread_block_size,
tiled_mma_t,
128,
value_type,
value_type,
value_type>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm4_Half_MMA_SwizzledSmemLayouts) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 128;
constexpr uint32_t n = 128;
constexpr uint32_t k = 128;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x16_F16F16F16F16_TN>,
Layout<Shape<_2, _2, _1>>, // 2x2x1 thread group
Tile<_32, _32, _16> // 32x32x16 MMA for LDSM, 1x2x1 value group`
>;
// RowMajor
using smem_rowmajor_atom_layout_t = decltype(
composition(Swizzle<3,3,3>{},
Layout<Shape < _8,_64>,
Stride<_64, _1>>{}));
// ColMajor
using smem_colmajor_atom_layout_t = decltype(
composition(Swizzle<3,3,3>{},
Layout<Shape <_64, _8>,
Stride< _1,_64>>{}));
using smem_a_atom_layout_t = smem_rowmajor_atom_layout_t;
using smem_b_atom_layout_t = smem_colmajor_atom_layout_t;
using smem_c_atom_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{}));
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<k> {}), GenRowMajor{}));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<n> {}, Int<k> {}), GenColMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<n> {}), GenRowMajor{}));
using smem_a_atom_layout_t = smem_a_atom_layout_t;
using smem_a_layout_t = decltype(tile_to_shape(
smem_a_atom_layout_t{},
make_shape(shape<0>(gmem_a_layout_t{}), shape<1>(gmem_a_layout_t{})))
);
using smem_b_atom_layout_t = smem_b_atom_layout_t;
using smem_b_layout_t = decltype(tile_to_shape(
smem_b_atom_layout_t{},
make_shape(shape<0>(gmem_b_layout_t{}), shape<1>(gmem_b_layout_t{})))
);
using smem_c_atom_layout_t = smem_c_atom_layout_t;
using smem_c_layout_t = decltype(tile_to_shape(
smem_c_atom_layout_t{},
make_shape(shape<0>(gmem_c_layout_t{}), shape<1>(gmem_c_layout_t{})))
);
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
SM75_U32x4_LDSM_N, // A
SM75_U16x8_LDSM_T, // B
AutoVectorizingCopyWithAssumedAlignment<128>, // C
thread_block_size,
tiled_mma_t,
128,
value_type,
value_type,
value_type>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm5_Double_MMA_SwizzledSmemLayouts) {
using value_type = double;
constexpr uint32_t m = 128;
constexpr uint32_t n = 64;
constexpr uint32_t k = 16;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<MMA_Atom<SM80_8x8x4_F64F64F64F64_TN>, // Atom
Layout<Shape<_2, _2, _1>>, // Atom layout
Tile<Layout<Shape<_16, _2>, Stride<_2, _1>>, // 32x32x4 MMA with perm for load vectorization
Layout<Shape<_16, _2>, Stride<_2, _1>>,
Underscore>>;
using smem_a_atom_layout_t = decltype(
composition(Swizzle<2,2,2>{},
Layout<Shape <_16, _4>,
Stride< _1,_16>>{})); // M, K
using smem_b_atom_layout_t = decltype(
composition(Swizzle<2,2,2>{},
Layout<Shape <_16, _4>,
Stride< _1,_16>>{})); // N, K
using smem_c_atom_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{}));
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<k> {}), GenRowMajor{}));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<n> {}, Int<k> {}), GenColMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<n> {}), GenRowMajor{}));
using smem_a_atom_layout_t = smem_a_atom_layout_t;
using smem_a_layout_t = decltype(tile_to_shape(
smem_a_atom_layout_t{},
make_shape(shape<0>(gmem_a_layout_t{}), shape<1>(gmem_a_layout_t{})))
);
using smem_b_atom_layout_t = smem_b_atom_layout_t;
using smem_b_layout_t = decltype(tile_to_shape(
smem_b_atom_layout_t{},
make_shape(shape<0>(gmem_b_layout_t{}), shape<1>(gmem_b_layout_t{})))
);
using smem_c_atom_layout_t = smem_c_atom_layout_t;
using smem_c_layout_t = decltype(tile_to_shape(
smem_c_atom_layout_t{},
make_shape(shape<0>(gmem_c_layout_t{}), shape<1>(gmem_c_layout_t{})))
);
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
AutoVectorizingCopyWithAssumedAlignment<128>, // A
AutoVectorizingCopyWithAssumedAlignment<128>, // B
AutoVectorizingCopyWithAssumedAlignment<128>, // C
thread_block_size,
tiled_mma_t,
128,
value_type,
value_type,
value_type>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm6_MixedPrecisionFP16FP32_MMA) {
using TA = cutlass::half_t;
using TB = cutlass::half_t;
using TC = float;
constexpr uint32_t m = 64;
constexpr uint32_t n = 64;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x8_F32F16F16F32_TN>,
Layout<Shape<_2, _2, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 128, TA, TB, TC>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm7_MixedPrecisionBF16FP32_MMA) {
using TA = cutlass::bfloat16_t;
using TB = cutlass::bfloat16_t;
using TC = float;
constexpr uint32_t m = 64;
constexpr uint32_t n = 64;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x8_F32BF16BF16F32_TN>,
Layout<Shape<_2, _2, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 128, TA, TB, TC>();
}
TEST(SM80_CuTe_Ampere, CooperativeGemm8_MixedPrecisionTF32FP32_MMA) {
using TA = cutlass::tfloat32_t;
using TB = cutlass::tfloat32_t;
using TC = float;
constexpr uint32_t m = 64;
constexpr uint32_t n = 64;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM80_16x8x8_F32TF32TF32F32_TN>,
Layout<Shape<_2, _2, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 128, TA, TB, TC>();
}

414
test/unit/cute/cooperative_gemm_common.hpp Normal file
View File

@@ -0,0 +1,414 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include "cutlass_unit_test.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include <iostream>
#include <thrust/host_vector.h>
#include <thrust/device_vector.h>
#include <cute/tensor.hpp>
using namespace cute;
template<class ALayout,
class BLayout,
class CLayout,
class SMemALayout,
class SMemBLayout,
class SMemCLayout,
class SmemCopyOpA,
class SmemCopyOpB,
class SmemCopyOpC,
uint32_t ThreadBlockSize,
class TiledMma,
uint32_t CopyMaxVecBits,
class TA,
class TB,
class TC,
class Alpha,
class Beta,
class ALoadTransform,
class BLoadTransform,
class CLoadTransform,
class CStoreTransform>
__launch_bounds__(ThreadBlockSize) __global__ void
cooperative_gemm_kernel(TA const* a,
TB const* b,
TC* c,
TC* c_out,
Alpha const alpha,
Beta const beta,
ALoadTransform a_load_transform,
BLoadTransform b_load_transform,
CLoadTransform c_load_transform,
CStoreTransform c_store_transform)
{
using namespace cute;
Tensor g_a_tensor = make_tensor(make_gmem_ptr(a), ALayout{});
Tensor g_b_tensor = make_tensor(make_gmem_ptr(b), BLayout{});
Tensor g_c_tensor = make_tensor(make_gmem_ptr(c), CLayout{});
Tensor g_c_out_tensor = make_tensor(make_gmem_ptr(c_out), CLayout{});
constexpr uint32_t copy_max_vec_bytes = CopyMaxVecBits / 8;
extern __shared__ float4 smem_buf[];
auto* smem_ptr = reinterpret_cast<unsigned char*>(smem_buf);
auto* smem_ptr_a = smem_ptr;
auto* smem_ptr_b = smem_ptr_a + round_up((sizeof(TA) * cosize(SMemALayout {})), copy_max_vec_bytes);
auto* smem_ptr_c = smem_ptr_b + round_up((sizeof(TB) * cosize(SMemBLayout {})), copy_max_vec_bytes);
Tensor s_a_tensor = make_tensor(make_smem_ptr<TA>(smem_ptr_a), SMemALayout{});
Tensor s_b_tensor = make_tensor(make_smem_ptr<TB>(smem_ptr_b), SMemBLayout{});
Tensor s_c_tensor = make_tensor(make_smem_ptr<TC>(smem_ptr_c), SMemCLayout{});
cooperative_copy<ThreadBlockSize, CopyMaxVecBits>(threadIdx.x, g_a_tensor, s_a_tensor);
cooperative_copy<ThreadBlockSize, CopyMaxVecBits>(threadIdx.x, g_b_tensor, s_b_tensor);
cooperative_copy<ThreadBlockSize, CopyMaxVecBits>(threadIdx.x, g_c_tensor, s_c_tensor);
cp_async_fence();
cp_async_wait<0>();
__syncthreads();
TiledMma tiled_mma;
cooperative_gemm<SmemCopyOpA, SmemCopyOpB, SmemCopyOpC>(
threadIdx.x, tiled_mma,
alpha, s_a_tensor, s_b_tensor, beta, s_c_tensor,
a_load_transform, b_load_transform, c_load_transform, c_store_transform
);
__syncthreads();
cooperative_copy<ThreadBlockSize, CopyMaxVecBits>(threadIdx.x, s_c_tensor, g_c_out_tensor);
}
template<class ALayout, // logical shape (M, K)
class BLayout, // logical shape (N, K)
class CLayout, // logical shape (M, N)
class SMemALayout, // logical shape (M, K)
class SMemBLayout, // logical shape (N, K)
class SMemCLayout, // logical shape (M, N)
class SmemCopyOpA,
class SmemCopyOpB,
class SmemCopyOpC,
uint32_t ThreadBlockSize,
class TiledMma,
uint32_t CopyMaxVecBits,
class TA,
class TB,
class TC,
class ALoadTransform = cute::identity,
class BLoadTransform = cute::identity,
class CLoadTransform = cute::identity,
class CStoreTransform = cute::identity>
void test_cooperative_gemm(ALoadTransform const& a_load_transform = {},
BLoadTransform const& b_load_transform = {},
CLoadTransform const& c_load_transform = {},
CStoreTransform const& c_store_transform = {})
{
using gmem_a_layout_t = ALayout;
using gmem_b_layout_t = BLayout;
using gmem_c_layout_t = CLayout;
using smem_a_layout_t = SMemALayout;
using smem_b_layout_t = SMemBLayout;
using smem_c_layout_t = SMemCLayout;
static_assert(size<0>(gmem_a_layout_t{}) == size<0>(gmem_c_layout_t{})); // AM == CM
static_assert(size<0>(gmem_b_layout_t{}) == size<1>(gmem_c_layout_t{})); // BN == CN
static_assert(size<1>(gmem_a_layout_t{}) == size<1>(gmem_b_layout_t{})); // AK == BK
static_assert(size<0>(smem_a_layout_t{}) == size<0>(smem_c_layout_t{})); // AM == CM
static_assert(size<0>(smem_b_layout_t{}) == size<1>(smem_c_layout_t{})); // BN == CN
static_assert(size<1>(smem_a_layout_t{}) == size<1>(smem_b_layout_t{})); // AK == BK
static_assert(cute::size(gmem_a_layout_t {}) == cute::size(smem_a_layout_t {}));
static_assert(cute::size(gmem_b_layout_t {}) == cute::size(smem_b_layout_t {}));
static_assert(cute::size(gmem_c_layout_t {}) == cute::size(smem_c_layout_t {}));
#if 0
print(" "); print("gmem: "); print(gmem_layout_t{}); print("\n");
print(" "); print("smem: "); print(smem_layout_t{}); print("\n");
print(" "); print("threads: "); print(ThreadBlockSize); print("\n");
#endif
const auto alpha = static_cast<TC>(1.1);
const auto beta = static_cast<TC>(1.2);
thrust::host_vector<TA> h_a(cosize(gmem_a_layout_t{}));
thrust::host_vector<TB> h_b(cosize(gmem_b_layout_t{}));
thrust::host_vector<TC> h_c(cosize(gmem_c_layout_t{}));
thrust::host_vector<TC> h_c_out(cosize(gmem_c_layout_t{}));
auto h_a_tensor = make_tensor(h_a.data(), gmem_a_layout_t{});
auto h_b_tensor = make_tensor(h_b.data(), gmem_b_layout_t{});
auto h_c_tensor = make_tensor(h_c.data(), gmem_c_layout_t{});
size_t max_size = std::max<size_t>({static_cast<size_t>(size(gmem_a_layout_t {})),
static_cast<size_t>(size(gmem_b_layout_t {})),
static_cast<size_t>(size(gmem_c_layout_t {}))});
for (size_t i = 0; i < max_size; ++i) {
double di = static_cast<double>(i);
if(i < size(gmem_a_layout_t{})) {
h_a_tensor(i) = static_cast<TA>(di / size(gmem_a_layout_t{}));
}
if(i < size(gmem_b_layout_t{})) {
h_b_tensor(i) = static_cast<TA>(di / size(gmem_a_layout_t{}));
}
if(i < size(gmem_c_layout_t{})) {
h_c_tensor(i) = static_cast<TC>((di*di) / size(gmem_a_layout_t{}));
}
}
thrust::device_vector<TA> d_a(h_a);
thrust::device_vector<TB> d_b(h_b);
thrust::device_vector<TC> d_c(h_c);
thrust::device_vector<TC> d_c_out(h_c_out.size(), TC(float(-1)));
const size_t shared_memory_size =
(sizeof(TA) * h_a.size()) + (sizeof(TB) * h_b.size()) + (sizeof(TC) * h_c.size());
auto kernel = cooperative_gemm_kernel<
gmem_a_layout_t, gmem_b_layout_t, gmem_c_layout_t,
smem_a_layout_t, smem_b_layout_t, smem_c_layout_t,
SmemCopyOpA, SmemCopyOpB, SmemCopyOpC,
ThreadBlockSize, TiledMma, CopyMaxVecBits,
TA, TB, TC, decltype(alpha), decltype(beta),
ALoadTransform, BLoadTransform, CLoadTransform, CStoreTransform
>;
ASSERT_EQ(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(shared_memory_size)), 0);
kernel<<<1, ThreadBlockSize, shared_memory_size>>>(
thrust::raw_pointer_cast(d_a.data()),
thrust::raw_pointer_cast(d_b.data()),
thrust::raw_pointer_cast(d_c.data()),
thrust::raw_pointer_cast(d_c_out.data()),
alpha,
beta,
a_load_transform,
b_load_transform,
c_load_transform,
c_store_transform
);
cudaError_t result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
cudaError_t error = cudaGetLastError();
FAIL() << "Error at kernel sync: " << cudaGetErrorString(error) << "\n";
}
thrust::host_vector<TC> h_c_ref(h_c.size(), static_cast<TC>(0.0));
auto h_c_ref_tensor = make_tensor(h_c_ref.data(), gmem_c_layout_t{});
// A * B
for (int k = 0; k < size<1>(h_a_tensor); k++) {
for (int m = 0; m < size<0>(h_a_tensor); m++) {
for (int n = 0; n < size<0>(h_b_tensor); n++) {
const auto a_value = a_load_transform(h_a_tensor(m, k));
const auto b_value = b_load_transform(h_b_tensor(n, k));
const auto a_value_fp64 = static_cast<double>(a_value);
const auto b_value_fp64 = static_cast<double>(b_value);
h_c_ref_tensor(m, n) += static_cast<TC>(a_value_fp64 * b_value_fp64);
}
}
}
// C = A*B + C
for (int i = 0; i < size(h_c_ref_tensor); i++) {
const auto ab_value_fp64 = static_cast<double>(h_c_ref_tensor(i));
const auto c_value_fp64 = static_cast<double>(c_load_transform(h_c_tensor(i)));
h_c_ref_tensor(i) = c_store_transform(static_cast<TC>(alpha * ab_value_fp64 + beta * c_value_fp64));
}
h_c_out = d_c_out;
auto h_c_out_tensor = make_tensor(h_c_out.data(), gmem_c_layout_t{});
for (int i = 0; i < size(h_c_ref_tensor); i++) {
double h_c_ref_i = h_c_ref_tensor(i);
double h_c_out_i = h_c_out_tensor(i);
double epsilon(0.1f);
double nonzero_floor(std::numeric_limits<double>::min());
bool passed = cutlass::relatively_equal(h_c_out_i, h_c_ref_i, epsilon, nonzero_floor);
ASSERT_TRUE(passed) << i << " - result:" << h_c_out_i << " expected:" << h_c_ref_i;
}
}
template<uint32_t M,
uint32_t N,
uint32_t K,
uint32_t ThreadBlockSize,
class TiledMMAType,
uint32_t CopyMaxVecBits,
class TA,
class TB,
class TC,
class ALoadTransform = cute::identity,
class BLoadTransform = cute::identity,
class CLoadTransform = cute::identity,
class CStoreTransform = cute::identity>
void test_cooperative_gemm_col_major_layout(ALoadTransform const& a_load_transform = {},
BLoadTransform const& b_load_transform = {},
CLoadTransform const& c_load_transform = {},
CStoreTransform const& c_store_transform = {})
{
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<M> {}, Int<K> {})));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<N> {}, Int<K> {}), GenRowMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<M> {}, Int<N> {})));
using smem_a_layout_t = decltype(make_layout(make_shape(Int<M> {}, Int<K> {})));
using smem_b_layout_t = decltype(make_layout(make_shape(Int<N> {}, Int<K> {}), GenRowMajor{}));
using smem_c_layout_t = decltype(make_layout(make_shape(Int<M> {}, Int<N> {})));
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
AutoVectorizingCopyWithAssumedAlignment<sizeof_bits_v<TA>>,
AutoVectorizingCopyWithAssumedAlignment<sizeof_bits_v<TB>>,
AutoVectorizingCopyWithAssumedAlignment<sizeof_bits_v<TC>>,
ThreadBlockSize,
TiledMMAType,
CopyMaxVecBits,
TA,
TB,
TC>(a_load_transform, b_load_transform, c_load_transform, c_store_transform);
}
template<uint32_t M,
uint32_t N,
uint32_t K,
uint32_t ThreadBlockSize,
class TiledMMAType,
class T,
class ALoadTransform = cute::identity,
class BLoadTransform = cute::identity,
class CLoadTransform = cute::identity,
class CStoreTransform = cute::identity>
void test_cooperative_gemm_col_major_layout(ALoadTransform const& a_load_transform = {},
BLoadTransform const& b_load_transform = {},
CLoadTransform const& c_load_transform = {},
CStoreTransform const& c_store_transform = {})
{
test_cooperative_gemm_col_major_layout<M, N, K, ThreadBlockSize, TiledMMAType, cute::sizeof_bits_v<T>, T, T, T>(
a_load_transform, b_load_transform, c_load_transform, c_store_transform);
}
template<class SMemAAtomLayout,
class SMemBAtomLayout,
class SMemCAtomLayout,
uint32_t M,
uint32_t N,
uint32_t K,
uint32_t ThreadBlockSize,
class TiledMMAType,
uint32_t CopyMaxVecBits,
class TA,
class TB,
class TC,
class ALoadTransform = cute::identity,
class BLoadTransform = cute::identity,
class CLoadTransform = cute::identity,
class CStoreTransform = cute::identity>
void test_cooperative_gemm_col_major_layout(ALoadTransform const& a_load_transform = {},
BLoadTransform const& b_load_transform = {},
CLoadTransform const& c_load_transform = {},
CStoreTransform const& c_store_transform = {})
{
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<M> {}, Int<K> {})));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<N> {}, Int<K> {}), GenRowMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<M> {}, Int<N> {})));
using smem_a_atom_layout_t = SMemAAtomLayout;
using smem_a_layout_t = decltype(tile_to_shape(
smem_a_atom_layout_t{},
make_shape(shape<0>(gmem_a_layout_t{}), shape<1>(gmem_a_layout_t{})))
);
using smem_b_atom_layout_t = SMemBAtomLayout;
using smem_b_layout_t = decltype(tile_to_shape(
smem_b_atom_layout_t{},
make_shape(shape<0>(gmem_b_layout_t{}), shape<1>(gmem_b_layout_t{})))
);
using smem_c_atom_layout_t = SMemCAtomLayout;
using smem_c_layout_t = decltype(tile_to_shape(
smem_c_atom_layout_t{},
make_shape(shape<0>(gmem_c_layout_t{}), shape<1>(gmem_c_layout_t{})))
);
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
AutoVectorizingCopyWithAssumedAlignment<sizeof_bits_v<TA>>,
AutoVectorizingCopyWithAssumedAlignment<sizeof_bits_v<TB>>,
AutoVectorizingCopyWithAssumedAlignment<sizeof_bits_v<TC>>,
ThreadBlockSize,
TiledMMAType,
CopyMaxVecBits,
TA,
TB,
TC>(a_load_transform, b_load_transform, c_load_transform, c_store_transform);
}
template<class SMemAAtomLayout,
class SMemBAtomLayout,
class SMemCAtomLayout,
uint32_t M,
uint32_t N,
uint32_t K,
uint32_t ThreadBlockSize,
class TiledMMAType,
class T,
class ALoadTransform = cute::identity,
class BLoadTransform = cute::identity,
class CLoadTransform = cute::identity,
class CStoreTransform = cute::identity>
void test_cooperative_gemm_col_major_layout(ALoadTransform const& a_load_transform = {},
BLoadTransform const& b_load_transform = {},
CLoadTransform const& c_load_transform = {},
CStoreTransform const& c_store_transform = {})
{
test_cooperative_gemm_col_major_layout<SMemAAtomLayout,
SMemBAtomLayout,
SMemCAtomLayout,
M,
N,
K,
ThreadBlockSize,
TiledMMAType,
cute::sizeof_bits_v<T>,
T,
T,
T>(a_load_transform, b_load_transform, c_load_transform, c_store_transform);
}

35
test/unit/cute/core/complement.cpp
View File

@@ -35,22 +35,22 @@
#include <cute/tensor.hpp>
template <class Layout, class CoSizeHi>
template <class Layout, class CoTarget>
void
test_complement(Layout const& layout, CoSizeHi const& cosize_hi)
test_complement(Layout const& layout, CoTarget const& cotarget)
{
using namespace cute;
auto result = complement(layout, cosize_hi);
auto result = complement(layout, cotarget);
CUTLASS_TRACE_HOST("complement(" << layout << ", " << cosize_hi << ") => " << result);
CUTLASS_TRACE_HOST("complement(" << layout << ", " << cotarget << ") => " << result);
auto completed = make_layout(layout, result);
// Lower-bound on the codomain size of the layout ++ complement (1)
EXPECT_GE(cosize(completed), cosize_hi);
EXPECT_GE(cosize(completed), size(cotarget));
// Upper-bound on the codomain size of the complement (2)
EXPECT_LE(cosize(result), cute::round_up(cosize_hi, cosize(layout)));
EXPECT_LE(cosize(result), cute::round_up(size(cotarget), cosize(layout)));
// Post-condition on the codomain of the complement
for (int i = 1; i < size(result); ++i) {
@@ -62,9 +62,9 @@ test_complement(Layout const& layout, CoSizeHi const& cosize_hi)
// Other observations
EXPECT_LE(size(result), cosize(result)); // As a result of the ordered condition (3)
EXPECT_GE(size(result), cosize_hi / size(filter(layout)));
EXPECT_GE(size(result), size(cotarget) / size(filter(layout)));
EXPECT_LE(cosize(completed), cosize(result) + cosize(layout));
EXPECT_GE(cosize(result), cosize_hi / size(filter(layout)));
EXPECT_GE(cosize(result), size(cotarget) / size(filter(layout)));
if constexpr (is_static<decltype(stride(completed))>::value) { // If we can apply complement again
EXPECT_EQ(size(complement(completed)), 1); // There's no more codomain left over
}
@@ -90,6 +90,8 @@ TEST(CuTe_core, Complement)
test_complement(layout);
test_complement(layout, Int<2>{});
test_complement(layout, Int<5>{});
test_complement(layout, make_shape(Int<2>{}, 2));
}
{
@@ -97,6 +99,8 @@ TEST(CuTe_core, Complement)
test_complement(layout);
test_complement(layout, Int<2>{});
test_complement(layout, Int<5>{});
test_complement(layout, make_shape(Int<2>{}, 2));
}
{
@@ -105,6 +109,8 @@ TEST(CuTe_core, Complement)
test_complement(layout, Int<1>{});
test_complement(layout, Int<2>{});
test_complement(layout, Int<8>{});
test_complement(layout, Int<5>{});
test_complement(layout, make_shape(Int<2>{}, 2));
}
{
@@ -130,6 +136,7 @@ TEST(CuTe_core, Complement)
test_complement(layout);
test_complement(layout, Int<16>{});
test_complement(layout, Int<19>{});
test_complement(layout, make_shape(Int<2>{}, 2));
}
{
@@ -138,6 +145,7 @@ TEST(CuTe_core, Complement)
test_complement(layout, Int<1>{});
test_complement(layout);
test_complement(layout, Int<17>{});
test_complement(layout, make_shape(Int<2>{}, 2));
}
{
@@ -193,8 +201,8 @@ TEST(CuTe_core, Complement)
// Fails due to non-injective layout
// {
// auto layout = make_layout(Shape<Shape<_2,_2>,Shape<_2, _2>>{},
// Stride<Stride<_1,_8>,Stride<_8,_4>>{});
// auto layout = make_layout(Shape <Shape <_2,_2>,Shape <_2,_2>>{},
// Stride<Stride<_1,_8>,Stride<_8,_4>>{});
// test_complement(layout);
// }
@@ -289,4 +297,11 @@ TEST(CuTe_core, Complement)
test_complement(layout);
}
{
auto layout = make_layout(Int<64>{});
test_complement(layout, make_shape(Int<32>{}, Int<4>{}, Int<4>{}));
test_complement(layout, make_shape(Int<32>{}, Int<4>{}, 4));
}
}

39
test/unit/cute/core/composition.cpp
View File

@@ -212,13 +212,12 @@ TEST(CuTe_core, Composition)
test_composition(a, b);
}
// FAILS due to b not "dividing into" a properly
//{
// auto a = make_layout(Shape<_4,_3>{});
// auto b = make_layout(Shape<_6>{});
{
auto a = make_layout(Shape<_4,_3>{});
auto b = make_layout(Shape<_6>{});
// test_composition(a, b);
//}
test_composition(a, b);
}
{
auto a = make_layout(Shape<_4,_3>{});
@@ -234,13 +233,12 @@ TEST(CuTe_core, Composition)
test_composition(a, b);
}
// FAILS due to b not "dividing into" a properly
//{
// auto a = make_layout(Shape<_4,_3>{});
// auto b = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
{
auto a = make_layout(Shape<_4,_3>{});
auto b = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
// test_composition(a, b);
//}
test_composition(a, b);
}
{
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
@@ -523,4 +521,21 @@ TEST(CuTe_core, Composition)
test_composition(a, b);
}
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("BETA: Tuple strides" );
CUTLASS_TRACE_HOST("-------------------------------");
{
auto a = make_layout(Shape<_4,_4>{}, Stride<_4,_1>{});
auto b = make_layout(Shape<_4,_4>{}, Stride<E<1>,E<0>>{});
test_composition(a, b);
}
{
auto a = make_layout(Shape<_4,Shape<_2,_3>>{}, Stride<_6,Stride<_3,_1>>{});
auto b = make_layout(Shape<_2,_4>{}, Stride<E<1,1>,E<0>>{});
test_composition(a, b);
}
}

51
test/unit/cute/core/logical_divide.cpp
View File

@@ -227,27 +227,42 @@ TEST(CuTe_core, Logical_divide)
ASSERT_TRUE(decltype(stride<1>(result) == Int<48>{})::value);
}
// DISALLOWED
//{
//auto layout = make_layout(make_shape(128,4,3), make_stride(1,512,0));
//auto tile = Layout<_32>{};
{
auto layout = make_layout(make_shape(Int<32>{}, Int<4>{}, 4));
auto tile = Layout<_64>{};
//test_logical_divide(layout, tile);
//}
test_logical_divide(layout, tile);
//{
//auto layout = make_layout(make_shape(128,4,3), make_stride(1,512,0));
//auto tile = Layout<_32,_2>{};
// Enforcement of result
auto result = logical_divide(layout, tile);
ASSERT_TRUE(bool( shape(result) == make_shape (_64{}, make_shape ( _2{}, 4))));
ASSERT_TRUE(bool(stride(result) == make_stride( _1{}, make_stride(_64{},_128{}))));
}
//CUTLASS_TRACE_HOST("complement: " << complement(tile, size(layout)));
//test_logical_divide(layout, tile);
//}
//{
//auto layout = make_layout(make_shape(16,4,3), make_stride(1,512,0));
//auto tile = Layout<_32>{};
//
// ALLOWED, but dangerous due to the dynamic lhs shapes
// Consider disallowing...
//
//CUTLASS_TRACE_HOST("complement: " << complement(tile, size(layout)));
//test_logical_divide(layout, tile);
//}
{
auto layout = make_layout(make_shape(128,4,3), make_stride(1,512,0));
auto tile = Layout<_32>{};
test_logical_divide(layout, tile);
}
{
auto layout = make_layout(make_shape(128,4,3), make_stride(1,512,0));
auto tile = Layout<_32,_2>{};
test_logical_divide(layout, tile);
}
{
auto layout = make_layout(make_shape(16,4,3), make_stride(1,512,0));
auto tile = Layout<_32>{};
test_logical_divide(layout, tile);
}
}

5
test/unit/cute/hopper/CMakeLists.txt
View File

@@ -56,6 +56,11 @@ cutlass_test_unit_add_executable(
tma_load.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_cute_hopper_tma_mcast_load
tma_mcast_load.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_cute_hopper_tma_store
tma_store.cu

2
test/unit/cute/hopper/tma_load.cu
View File

@@ -44,7 +44,6 @@ test_tma_load(GMEM_Layout const& gmem_layout,
SMEM_Layout const& smem_layout,
CTA_Tile const& cta_tile)
{
using namespace cute;
return test_tma_load<T, TmaType>(SM90_TMA_LOAD{}, gmem_layout, smem_layout, cta_tile);
}
@@ -53,7 +52,6 @@ auto
test_tma_load(GMEM_Layout const& gmem_layout,
SMEM_Layout const& smem_layout)
{
using namespace cute;
return test_tma_load<T, TmaType>(gmem_layout, smem_layout, product_each(shape(smem_layout)));
}

76
test/unit/cute/hopper/tma_mcast_load.cu Normal file
View File

@@ -0,0 +1,76 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "../hopper/tma_mcast_load_testbed.hpp"
using namespace cute;
using namespace cutlass::test;
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
template <class T, class TmaType = T, class GMEM_Layout, class SMEM_Layout, class CTA_Tile, class Cluster_Size = Int<1>>
auto
test_tma_load(GMEM_Layout const& gmem_layout,
SMEM_Layout const& smem_layout,
CTA_Tile const& cta_tile,
Cluster_Size const& cluster_size = {})
{
return test_tma_load<T, TmaType>(SM90_TMA_LOAD_MULTICAST{}, gmem_layout, smem_layout, cta_tile, cluster_size);
}
template <class T, class TmaType = T, class GMEM_Layout, class SMEM_Layout>
auto
test_tma_load(GMEM_Layout const& gmem_layout,
SMEM_Layout const& smem_layout)
{
return test_tma_load<T, TmaType>(gmem_layout, smem_layout, product_each(shape(smem_layout)));
}
TEST(SM90_CuTe_Hopper, Tma_Load_32x32_Col_MCast)
{
Layout smem_layout = Layout<Shape<_32,_32>, Stride<_1,_32>>{};
{
Layout gmem_layout = make_layout(make_shape(32,32), GenColMajor{});
test_tma_load<int8_t>(gmem_layout, smem_layout, shape(smem_layout), Int<2>{});
test_tma_load<half_t>(gmem_layout, smem_layout, shape(smem_layout), Int<2>{});
test_tma_load< float>(gmem_layout, smem_layout, shape(smem_layout), Int<2>{});
test_tma_load<double>(gmem_layout, smem_layout, shape(smem_layout), Int<2>{});
test_tma_load<int8_t>(gmem_layout, smem_layout, shape(smem_layout), 2);
test_tma_load<half_t>(gmem_layout, smem_layout, shape(smem_layout), 2);
test_tma_load< float>(gmem_layout, smem_layout, shape(smem_layout), 2);
test_tma_load<double>(gmem_layout, smem_layout, shape(smem_layout), 2);
}
}
#endif

242
test/unit/cute/hopper/tma_mcast_load_testbed.hpp Normal file
View File

@@ -0,0 +1,242 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include "cutlass_unit_test.h"
#include <iostream>
#include <cstdint>
#include <thrust/host_vector.h>
#include <thrust/device_vector.h>
#include <cute/tensor.hpp>
#include <cute/arch/cluster_sm90.hpp>
#include <cutlass/cluster_launch.hpp>
namespace cutlass::test {
template <class ElementType, class SmemLayout>
struct SharedStorage
{
cute::ArrayEngine<ElementType, cute::cosize_v<SmemLayout>> smem;
alignas(16) cute::uint64_t tma_load_mbar[1];
};
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
template <class T, class GmemLayout, class SmemLayout,
class CopyAtom, class CTA_Tiler, class Cluster_Size>
__global__ void
tma_test_device_cute(T const* g_in, T* g_out, GmemLayout gmem_layout, SmemLayout smem_layout,
CUTE_GRID_CONSTANT CopyAtom const tma, CTA_Tiler cta_tiler, Cluster_Size cluster_size)
{
using namespace cute;
CUTE_STATIC_ASSERT_V(product_each(shape(cta_tiler)) == product_each(shape(smem_layout)));
// Use Shared Storage structure to allocate and distribute aligned SMEM addresses
extern __shared__ char shared_memory[];
using SharedStorage = SharedStorage<T, SmemLayout>;
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
// Construct SMEM tensor
Tensor sA = make_tensor(make_smem_ptr(shared_storage.smem.begin()), smem_layout); // (CTA_TILE_M,CTA_TILE_N,...)
// Shared memory barriers use 64bits in SMEM for synchronization
uint64_t* tma_load_mbar = shared_storage.tma_load_mbar;
// TMA requires special handling of strides to deal with coord codomain mapping
// Represent the full tensors -- get these from TMA
Tensor mA = tma.get_tma_tensor(shape(gmem_layout));
Tensor mB = make_tensor(make_gmem_ptr<T>(g_out), gmem_layout);
Tensor gA = zipped_divide(mA, cta_tiler); // ((CTA_TILE_M,CTA_TILE_N,...),(REST_M,REST_N,...))
Tensor gB = zipped_divide(mB, cta_tiler); // ((CTA_TILE_M,CTA_TILE_N,...),(REST_M,REST_N,...))
#if 1
if (thread0()) {
print(tma);
print("TILE : "); print(cta_tiler); print("\n");
print(" mA : "); print( mA); print("\n");
print(" mB : "); print( mB); print("\n");
print(" gA : "); print( gA); print("\n");
print(" gB : "); print( gB); print("\n");
print(" sA : "); print( sA); print("\n");
} __syncthreads(); cute::cluster_sync();
#endif
//
// Prepare the TMA_LOAD
//
Tensor sA_x = make_tensor(sA.data(), make_layout(sA.layout(), Layout<_1>{})); // ((CTA_TILE_M,CTA_TILE_N,...),_1)
Tensor tBgB = gB; // ((CTA_TILE_M,CTA_TILE_N,...),(REST_M,REST_N,...))
int cta_rank_in_cluster = cute::block_rank_in_cluster();
auto [tAgA, tAsA] = tma_partition(tma, cta_rank_in_cluster, make_layout(cluster_size), sA_x, gA);
#if 1
if (thread0()) {
print("sA_x : "); print(sA_x); print("\n");
print("tBgB : "); print(tBgB); print("\n");
print("tAgA : "); print(tAgA); print("\n");
print("tAsA : "); print(tAsA); print("\n");
} __syncthreads(); cute::cluster_sync();
#endif
//
// TMA Multicast Masks -- Get a mask of the active ctas in each TMA
//
int elected_cta_rank = 0;
bool elect_one_cta = (elected_cta_rank == cta_rank_in_cluster);
bool elect_one_thr = cute::elect_one_sync();
uint16_t tma_mcast_mask = ((uint16_t(1) << cluster_size) - 1);
#if 1
if (thread0()) {
print("tma_mcast_mask : "); print(tma_mcast_mask); print("\n");
} __syncthreads(); cute::cluster_sync();
#endif
//
// Perform the TMA_LOAD
//
if (elect_one_thr) {
// Initialize TMA barrier
cute::initialize_barrier(tma_load_mbar[0], /* num_threads */ 1);
}
int tma_phase_bit = 0;
// Ensures all CTAs in the Cluster have initialized
__syncthreads();
cute::cluster_sync();
// Loop over the TMA stages, using smem as our buffer
for (int stage = 0; stage < size<1>(tAgA); ++stage)
{
// Set the bytes transferred in this TMA transaction (may involve multiple issues)
constexpr int kTmaTransactionBytes = sizeof(ArrayEngine<T, size(sA)>);
if (elect_one_thr)
{
cute::set_barrier_transaction_bytes(tma_load_mbar[0], kTmaTransactionBytes);
copy(tma.with(tma_load_mbar[0], tma_mcast_mask), tAgA(_,stage), tAsA(_,0));
}
__syncthreads();
/// Wait on the shared memory barrier until the phase bit flips from tma_phase_bit value
cute::wait_barrier(tma_load_mbar[0], tma_phase_bit);
tma_phase_bit ^= 1;
//
// Write out trivially smem -> gmem
//
// Subbyte elements could cause race conditions, so be even more conservative
if (elect_one_cta && elect_one_thr) {
copy(sA, tBgB(_,stage));
}
__syncthreads();
cute::cluster_sync();
}
}
template <class T, class TmaType = T, class CopyOp,
class GMEM_Layout, class SMEM_Layout,
class CTA_Tiler, class Cluster_Size>
auto
test_tma_load(CopyOp const& copy_op,
GMEM_Layout const& gmem_layout,
SMEM_Layout const& smem_layout,
CTA_Tiler const& cta_tiler,
Cluster_Size const& cluster_size)
{
using namespace cute;
// Allocate and initialize host test data
size_t N = ceil_div(cosize(gmem_layout) * sizeof_bits<T>::value, 8);
thrust::host_vector<uint8_t> h_in(N);
for (size_t i = 0; i < h_in.size(); ++i) {
h_in[i] = uint8_t(i % 13);
}
Tensor hA_in = make_tensor(recast_ptr<T>(h_in.data()), gmem_layout);
// Allocate and initialize device test data
thrust::device_vector<uint8_t> d_in = h_in;
thrust::device_vector<uint8_t> d_out(h_in.size(), uint8_t(-1)); // overflow uint
// Create TMA for this device Tensor
Tensor gA = make_tensor(make_gmem_ptr<T>(raw_pointer_cast(d_in.data())), gmem_layout);
auto tma = make_tma_atom<TmaType>(copy_op, gA, smem_layout, cta_tiler, cluster_size);
//print(tma);
// Launch
dim3 dimBlock(32);
dim3 dimCluster(size(cluster_size));
dim3 dimGrid = dimCluster;
int smem_size = sizeof(SharedStorage<T, SMEM_Layout>);
void* kernel_ptr = (void*) &tma_test_device_cute<T, GMEM_Layout, SMEM_Layout,
decltype(tma), CTA_Tiler, Cluster_Size>;
cutlass::launch_kernel_on_cluster({dimGrid, dimBlock, dimCluster, smem_size},
kernel_ptr,
reinterpret_cast<T const*>(raw_pointer_cast(d_in.data())),
reinterpret_cast<T *>(raw_pointer_cast(d_out.data())),
gmem_layout,
smem_layout,
tma, cta_tiler, cluster_size);
// Copy results back to host
thrust::host_vector<uint8_t> h_out = d_out;
Tensor hA_out = make_tensor(recast_ptr<T>(h_out.data()), gmem_layout);
// Validate the results. Print only the first 3 errors.
int count = 3;
for (int i = 0; i < int(size(hA_out)) && count > 0; ++i) {
EXPECT_EQ(hA_in(i), hA_out(i));
if (hA_in(i) != hA_out(i)) {
--count;
}
}
return tma;
}
#endif
} // end namespace cutlass::test

32
test/unit/cute/turing/CMakeLists.txt Normal file
View File

@@ -0,0 +1,32 @@
# Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cutlass_test_unit_add_executable(
cutlass_test_unit_cute_turing
cooperative_gemm.cu
)

58
test/unit/cute/turing/cooperative_gemm.cu Normal file
View File

@@ -0,0 +1,58 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include "../cooperative_gemm_common.hpp"
using namespace cute;
TEST(SM75_CuTe_Turing, CooperativeGemm1_MixedPrecisionFP16FP32_MMA) {
using TA = cutlass::half_t;
using TB = cutlass::half_t;
using TC = float;
constexpr uint32_t m = 64;
constexpr uint32_t n = 64;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<SM75_16x8x8_F32F16F16F32_TN>,
Layout<Shape<_2, _2, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 128, TA, TB, TC>();
}

1
test/unit/cute/volta/CMakeLists.txt
View File

@@ -30,4 +30,5 @@ cutlass_test_unit_add_executable(
cutlass_test_unit_cute_volta
vectorization_auto.cu
cooperative_copy.cu
cooperative_gemm.cu
)

48
test/unit/cute/volta/cooperative_copy.cu
View File

@@ -263,6 +263,21 @@ TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefault1D)
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefault1DFallback)
{
using value_type = float;
constexpr uint32_t count = 99;
using gmem_layout_t = decltype(make_layout(make_shape(Int<count>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<count>{})));
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2D)
{
using value_type = float;
@@ -279,6 +294,22 @@ TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2D)
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2DFallback)
{
using value_type = float;
constexpr uint32_t x = 37;
constexpr uint32_t y = 37;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{})));
constexpr uint32_t thread_block_size = 64;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2DCustomStride)
{
using value_type = float;
@@ -312,6 +343,23 @@ TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG3D)
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG3DFallback)
{
using value_type = cute::half_t;
constexpr uint32_t x = 44;
constexpr uint32_t y = 24;
constexpr uint32_t z = 14;
using gmem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{})));
using smem_layout_t = decltype(make_layout(make_shape(Int<x>{}, Int<y>{}, Int<z>{})));
constexpr uint32_t thread_block_size = 128;
test_cooperative_copy_default<typename TestFixture::mode,
TestFixture::max_vec_bits,
gmem_layout_t,
smem_layout_t,
thread_block_size,
value_type>();
}
TYPED_TEST(SM70_CuTe_Volta, CooperativeCopyDefaultGSSG2Dto3D)
{
using value_type = double;

421
test/unit/cute/volta/cooperative_gemm.cu Normal file
View File

@@ -0,0 +1,421 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cute/tensor.hpp>
#include "../cooperative_gemm_common.hpp"
using namespace cute;
TEST(SM70_CuTe_Volta, CooperativeGemm1_FloatFMA) {
using value_type = float;
constexpr uint32_t m = 64;
constexpr uint32_t n = 32;
constexpr uint32_t k = 16;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<UniversalFMA<value_type, value_type, value_type, value_type>>,
Layout<Shape<_16, _8, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm1_FloatFMA_Predication) {
using value_type = float;
constexpr uint32_t m = 88;
constexpr uint32_t n = 20;
constexpr uint32_t k = 12;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<UniversalFMA<value_type, value_type, value_type, value_type>>,
Layout<Shape<_2, _64, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm1_FloatFMA_Predication2) {
using value_type = float;
constexpr uint32_t m = 88;
constexpr uint32_t n = 36;
constexpr uint32_t k = 24;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<UniversalFMA<value_type, value_type, value_type, value_type>>,
Layout<Shape<_4, _32, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm1_FloatFMA_Predication3) {
using value_type = float;
constexpr uint32_t m = 67;
constexpr uint32_t n = 13;
constexpr uint32_t k = 11;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<UniversalFMA<value_type, value_type, value_type, value_type>>,
Layout<Shape<_1, _128, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm2_DoubleFMA) {
using value_type = double;
constexpr uint32_t m = 16;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t =
TiledMMA<
MMA_Atom<UniversalFMA<value_type, value_type, value_type, value_type>>,
Layout<Shape<_16, _8, _1>>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm3_Float_FMA_CustomPermutationMNK) {
using value_type = float;
constexpr uint32_t m = 32;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 256;
using tiled_mma_t = TiledMMA<
MMA_Atom<
UniversalFMA<value_type, value_type, value_type, value_type>
>,
Layout<
Shape<_16, _16, _1>
>,
Tile<
Layout<
Shape<_16,_2>, Stride<_2,_1>
>, // 32x32x1 MMA with perm for load vectorization
Layout<
Shape<_16,_2>, Stride<_2,_1>
>,
Underscore
>
>;
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm4_Half_MMA) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 32;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<SM70_8x8x4_F16F16F16F16_TN>,
Layout<Shape<_4, _4, _1>>
>;
using smem_a_atom_layout_t = typename tiled_mma_t::AtomLayoutB_TV;
using smem_b_atom_layout_t = typename tiled_mma_t::AtomLayoutA_TV;
using smem_c_atom_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<n> {})));
test_cooperative_gemm_col_major_layout<smem_a_atom_layout_t,
smem_b_atom_layout_t,
smem_c_atom_layout_t,
m,
n,
k,
thread_block_size,
tiled_mma_t,
value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm5_Half_MMA) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 32;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<SM70_8x8x4_F16F16F16F16_TN>,
Layout<Shape<_4, _4, _1>>
>;
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<k>{})));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<n>{}, Int<k>{}), GenColMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{})));
using smem_a_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<k>{})));
using smem_b_layout_t = decltype(make_layout(make_shape(Int<n>{}, Int<k>{}), GenColMajor{}));
using smem_c_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{})));
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
AutoVectorizingCopyWithAssumedAlignment<128>, // A
AutoVectorizingCopyWithAssumedAlignment<128>, // B
AutoVectorizingCopyWithAssumedAlignment<128>, // C
thread_block_size,
tiled_mma_t,
128,
value_type,
value_type,
value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm5_Half_MMA_Predicated) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 31;
constexpr uint32_t n = 27;
constexpr uint32_t k = 17;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<SM70_8x8x4_F16F16F16F16_TN>,
Layout<Shape<_4, _4, _1>>
>;
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<k>{})));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<n>{}, Int<k>{}), GenColMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{})));
using smem_a_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<k>{})));
using smem_b_layout_t = decltype(make_layout(make_shape(Int<n>{}, Int<k>{}), GenColMajor{}));
using smem_c_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{})));
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
AutoVectorizingCopyWithAssumedAlignment<16>, // A
AutoVectorizingCopyWithAssumedAlignment<16>, // B
AutoVectorizingCopyWithAssumedAlignment<16>, // C
thread_block_size,
tiled_mma_t,
16,
value_type,
value_type,
value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm6_Half_MAA_SwizzledSmemLayouts) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 128;
constexpr uint32_t n = 128;
constexpr uint32_t k = 64;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<SM70_8x8x4_F16F16F16F16_TN>,
Layout<Shape<_4, _4, _1>>
>;
using smem_a_atom_layout_t = decltype(
composition(Swizzle<3,3,3>{},
Layout<Shape < _8,_64>,
Stride<_64, _1>>{}));
using smem_b_atom_layout_t = decltype(
composition(Swizzle<3,3,3>{},
Layout<Shape <_64, _8>,
Stride< _1,_64>>{}));
using smem_c_atom_layout_t = decltype(make_layout(make_shape(Int<m>{}, Int<n>{}), GenRowMajor{}));
using gmem_a_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<k> {}), GenRowMajor{}));
using gmem_b_layout_t = decltype(make_layout(make_shape(Int<n> {}, Int<k> {}), GenColMajor{}));
using gmem_c_layout_t = decltype(make_layout(make_shape(Int<m> {}, Int<n> {}), GenRowMajor{}));
using smem_a_atom_layout_t = smem_a_atom_layout_t;
using smem_a_layout_t = decltype(tile_to_shape(
smem_a_atom_layout_t{},
make_shape(shape<0>(gmem_a_layout_t{}), shape<1>(gmem_a_layout_t{})))
);
// Transposed
using smem_b_atom_layout_t = smem_b_atom_layout_t;
using smem_b_layout_t = decltype(tile_to_shape(
smem_b_atom_layout_t{},
make_shape(shape<0>(gmem_b_layout_t{}), shape<1>(gmem_b_layout_t{})))
);
using smem_c_atom_layout_t = smem_c_atom_layout_t;
using smem_c_layout_t = decltype(tile_to_shape(
smem_c_atom_layout_t{},
make_shape(shape<0>(gmem_c_layout_t{}), shape<1>(gmem_c_layout_t{})))
);
test_cooperative_gemm<gmem_a_layout_t,
gmem_b_layout_t,
gmem_c_layout_t,
smem_a_layout_t,
smem_b_layout_t,
smem_c_layout_t,
AutoVectorizingCopyWithAssumedAlignment<128>, // A
AutoVectorizingCopyWithAssumedAlignment<128>, // B
AutoVectorizingCopyWithAssumedAlignment<128>, // C
thread_block_size,
tiled_mma_t,
128,
value_type,
value_type,
value_type>();
}
TEST(SM70_CuTe_Volta, CooperativeGemm7_TransformNegate_FMA) {
using TA = float;
using TB = float;
using TC = double;
constexpr uint32_t m = 32;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<UniversalFMA<TC, TA, TB, TC>>,
Layout<Shape<_16, _8, _1>>
>;
auto aload = cute::negate {};
auto bload = cute::negate {};
auto cload = cute::negate {};
auto cstore = cute::negate {};
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 64, TA, TB, TC>(
aload, bload, cload, cstore);
}
TEST(SM70_CuTe_Volta, CooperativeGemm7_TransformNegate_MMA) {
using value_type = cutlass::half_t;
constexpr uint32_t m = 32;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<SM70_8x8x4_F16F16F16F16_TN>,
Layout<Shape<_4, _4, _1>>
>;
auto aload = cute::negate {};
auto bload = cute::negate {};
auto cload = cute::negate {};
auto cstore = cute::negate {};
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, value_type>(
aload, bload, cload, cstore);
}
template<class ConstantType>
struct increment_by_x {
ConstantType x;
template <class T>
CUTE_HOST_DEVICE constexpr
T operator()(const T& arg) const {
return arg + x;
}
};
template<class From, class To>
struct convert_to {
CUTE_HOST_DEVICE constexpr
To operator()(const From& arg) const {
return static_cast<To>(arg);
}
};
TEST(SM70_CuTe_Volta, CooperativeGemm7_TransformCustomOp_FMA) {
using TA = float;
using TB = float;
using TC = double;
constexpr uint32_t m = 32;
constexpr uint32_t n = 32;
constexpr uint32_t k = 32;
constexpr uint32_t thread_block_size = 128;
using tiled_mma_t = TiledMMA<
MMA_Atom<UniversalFMA<TC, TA, TB, TC>>,
Layout<Shape<_16, _8, _1>>
>;
auto aload = increment_by_x<float>{1.111f};
auto bload = convert_to<float, double> {};
auto cload = cute::negate {};
auto cstore = cute::negate {};
test_cooperative_gemm_col_major_layout<m, n, k, thread_block_size, tiled_mma_t, 64, TA, TB, TC>(
aload, bload, cload, cstore);
}

2
test/unit/epilogue/thread/linear_combination_planar_complex.cu
View File

@@ -183,7 +183,7 @@ TEST(Epilogue_thread_linear_combination_planar_complex, f16_f32) {
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
}
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination{ linear_combination_op(accum, source) };
// Verify each result
for (int i = 0; i < kCount; ++i) {

13
test/unit/epilogue/threadblock/testbed.h
View File

@@ -42,6 +42,7 @@
#include "cutlass/half.h"
#include "cutlass/complex.h"
#include "cutlass/quaternion.h"
#include "cutlass/platform/platform.h"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "cutlass/util/host_tensor.h"
@@ -193,15 +194,15 @@ public:
cutlass::reference::host::TensorFillRandomUniform(
accumulator_tensor.host_view(),
seed,
20,
-20,
2,
-2,
0);
cutlass::reference::host::TensorFillRandomUniform(
source_tensor.host_view(),
seed + 2018,
20,
-20,
2,
-2,
0);
}
@@ -300,7 +301,9 @@ public:
output_params.alpha * ElementCompute(accumulator_tensor.at(coord)) +
output_params.beta * ElementCompute(source_tensor.at(coord));
if (std::numeric_limits<ElementOutput>::is_integer
if ((cutlass::platform::is_same<ElementOutput, cutlass::int4b_t>::value
|| cutlass::platform::is_same<ElementOutput, cutlass::uint4b_t>::value
|| std::numeric_limits<ElementOutput>::is_integer)
&& !std::numeric_limits<ElementCompute>::is_integer) {
std::fesetround(FE_TONEAREST);
expected = ElementOutput(std::nearbyint(float(cutlass::real(intermediate))));

1
test/unit/gemm/device/gemm_testbed_3x.hpp
View File

@@ -1336,7 +1336,6 @@ struct TestbedImpl {
{
using namespace cute;
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
auto mainloop_params = collective_mma_inputs.to_host_args(problem_size);
auto epilogue_params = collective_epilogue.to_host_args(problem_size);

44
test/unit/gemm/device/sm90_gemm_f32_f32_f32_tensor_op_f32.cu
View File

@@ -163,6 +163,50 @@ TEST(SM90_Device_Gemm_f32t_f32t_f32n_tensor_op_gmma_f32, 128x128x32_1x1x1_cooper
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
}
TEST(SM90_Device_Gemm_f32t_f32t_f32n_tensor_op_gmma_f32, 128x128x32_1x1x1_cooperative_narrow_wgmma) {
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::ColumnMajor;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
Shape<_128,_128,_32>, Shape<_1,_1,_1>,
cutlass::epilogue::collective::EpilogueTileAuto,
float, float,
float, LayoutC, 4,
float, LayoutC, 4,
cutlass::epilogue::TmaWarpSpecializedCooperative
>::CollectiveOp;
// Manually configure a half-tile wide MMA instruction
using CollectiveMainloop = cutlass::gemm::collective::CollectiveMma<
cutlass::gemm::MainloopSm90TmaGmmaWarpSpecialized<5, Shape<_1,_1,_1>, cutlass::gemm::KernelTmaWarpSpecializedCooperative>,
Shape<_128,_128,_32>,
float,
cutlass::detail::TagToStrideA_t<LayoutA>,
float,
cutlass::detail::TagToStrideB_t<LayoutB>,
decltype(cute::make_tiled_mma(cute::SM90_64x64x8_F32TF32TF32_SS_TN{}, Layout<Shape<_2,_1,_1>>{})),
cute::SM90_TMA_LOAD,
cute::GMMA::Layout_K_SW128_Atom<tfloat32_t>,
void,
cute::identity,
cute::SM90_TMA_LOAD,
cute::GMMA::Layout_K_SW128_Atom<tfloat32_t>,
void,
cute::identity
>;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
Shape<int,int,int,int>,
CollectiveMainloop,
CollectiveEpilogue
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
}
///////////////////////////////////////////////////////////////////////////////
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)

11
test/unit/gemm/device/sm90_gemm_f8_f8_f8_tensor_op_fp32.cu
View File

@@ -54,6 +54,7 @@
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
using namespace cute;
///////////////////////////////////////////////////////////////////////////////
//////////////////////////////// output: E4M3 /////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
@@ -760,7 +761,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_e5m2n_tensor_op_gmma_f32, 64x128x128_2x4x1_non
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
}
// Use Hopper FP8+AUX from 12.1
#if (!((__CUDACC_VER_MAJOR__ == 12) && (__CUDACC_VER_MINOR__ == 0)))
///////////////////////////////////////////////////////////////////////////////
///////////////////////// output: E4M3 + Aux Tensor ///////////////////////////
@@ -808,6 +810,7 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_e4m3n_tensor_op_gmma_f32, 64x128x128_aux_tenso
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
}
#endif
///////////////////////////////////////////////////////////////////////////////
////////////////////////////////// FP8 Accum /////////////////////////////////
@@ -990,6 +993,10 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_e4m3n_tensor_op_gmma_f32, 64x128x128_bias_bf16
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
}
// Use Hopper FP8+AUX from 12.1
#if (!((__CUDACC_VER_MAJOR__ == 12) && (__CUDACC_VER_MINOR__ == 0)))
///////////////////////////////////////////////////////////////////////////////
///////////////////// output: E4M3 + Aux Tensor + Bias/////////////////////////
///////////////////////////////////////////////////////////////////////////////
@@ -1142,6 +1149,8 @@ TEST(SM90_Device_Gemm_e4m3t_e5m2n_e4m3n_tensor_op_gmma_f32, 64x128x128_aux_tenso
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
}
#endif
///////////////////////////////////////////////////////////////////////////////
//////////////////////////////// TMA epilogue /////////////////////////////////
///////////////////////////////////////////////////////////////////////////////

2
test/unit/nvrtc/thread/testbed.h
View File

@@ -275,7 +275,7 @@ struct Testbed {
nvrtcAddNameExpression(program, gemm_kernel_instantiation.c_str());
const char *opts[] = {"--gpu-architecture=compute_75",
"--std=c++11",
"--std=c++17",
"--include-path=/usr/local/cuda-10.1/include"};
result_nvrtc = nvrtcCompileProgram(program, 3, opts);