leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

CUTLASS 2.10 (#615)

Browse Source 
Co-authored-by: Aniket Shivam <ashivam@nvidia.com>
This commit is contained in:
ANIKET SHIVAM
2022-09-03 15:48:46 -07:00
committed by GitHub
parent ca23ff7924
commit b72cbf957d
289 changed files with 43708 additions and 2513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
test/unit/conv/device/CMakeLists.txt
View File

@@ -110,6 +110,7 @@ cutlass_test_unit_add_executable(
# F16
conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f16nhwc_simt_f16_sm60.cu
depthwise_fprop_implicit_gemm_f16nhwc_f16nhwc_f16nhwc_simt_f16_sm60.cu
)
if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
@@ -177,12 +178,16 @@ if (CUTLASS_NVCC_MAX_ARCH GREATER_EQUAL 80)
# Conv2d (small channel count specializations)
conv2d_fprop_fixed_channels_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_sm80.cu
conv2d_fprop_few_channels_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_sm80.cu
# Conv2d (Strided Dgrad)
conv2d_strided_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
conv2d_strided_dgrad_implicit_gemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f32_sm80.cu
# Conv3d
conv3d_wgrad_implicit_gemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32_sm80.cu
# Group Conv2d
group_conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_sm80.cu
)
# Conv - TF32 input, F32 output, F32 accumulation

4
test/unit/conv/device/conv2d_fprop_few_channels_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_sm80.cu
View File

@@ -109,7 +109,7 @@ std::vector<cutlass::conv::Conv2dProblemSize> Conv2dFewChannelProblemSizes(int c
}
////////////////////////////////////////////////////////////////////////////////
#if 0
TEST(SM80_Device_Conv2d_Fprop_Few_Channels_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_channels_8,
128x128_64x3_64x64x64) {
@@ -201,7 +201,7 @@ TEST(SM80_Device_Conv2d_Fprop_Few_Channels_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<Conv2dFprop>(
Conv2dFewChannelProblemSizes(kChannelCount)));
}
#endif
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Conv2d_Fprop_Few_Channels_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_channels_2,

148
test/unit/conv/device/conv2d_problems.h
View File

@@ -684,6 +684,154 @@ struct TestbedConv2dProblemSizes {
};
////////////////////////////////////////////////////////////////////////////
/// Structure TestbedGroupConv2dProblemSizes initializes and holds group conv default and
/// important network sizes
////////////////////////////////////////////////////////////////////////////
struct TestbedGroupConv2dProblemSizes {
//
// Data members
//
int threadblock_n;
int threadblock_k;
int minimum_channel_size;
Conv2dProblemVector default_single_group_sizes;
Conv2dProblemVector default_multiple_group_sizes;
//
// Methods
//
/// Default ctor
TestbedGroupConv2dProblemSizes(
int threadblock_n_,
int threadblock_k_,
int minimum_channel_size_ = 64)
: threadblock_n (threadblock_n_),
threadblock_k (threadblock_k_),
minimum_channel_size (minimum_channel_size_) {
initialize_group_conv2d_default_sizes();
filter_all();
}
/// Eliminates some illegal cases
void filter_all() {
Conv2dProblemVector *problems_vectors[] = {
&default_single_group_sizes,
&default_multiple_group_sizes
};
for (Conv2dProblemVector *problems : problems_vectors) {
Conv2dProblemVector filtered;
for (cutlass::conv::Conv2dProblemSize const & problem : *problems) {
if (!((problem.C / problem.groups) % minimum_channel_size)) {
filtered.push_back(problem);
}
}
*problems = filtered;
}
}
// Add a few standard convolution problem sizes
void initialize_group_conv2d_default_sizes() {
////////////////////////////////////////////////////////////////////////////////////
// One group calculated by one or multiple CTAs: k_per_group % CTA::N = 0
// One CTA calculates a single group
////////////////////////////////////////////////////////////////////////////////////
for (int cta_per_group_k = 1; cta_per_group_k < 4; ++cta_per_group_k) {
// groups = 2, 3, 4
for (int groups = 2; groups < 5; ++groups) {
int conv_k = cta_per_group_k * threadblock_n * groups;
default_single_group_sizes.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, threadblock_k * 2 * groups}, // input size (NHWC)
{conv_k, 3, 3, threadblock_k * 2}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation,
1, // split_k_slices
groups // groups
));
} // loop groups
} // loop cta_per_group_k
// Partial gemm_k: k_per_group == CTA::N && channels_per_group < CTA::K
default_single_group_sizes.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, threadblock_k}, // input size (NHWC)
{threadblock_n * 2, 3, 3, threadblock_k / 2}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation,
1, // split_k_slices
2 // groups
));
////////////////////////////////////////////////////////////////////////////////////
// One CTA calculate multiple groups: CTA::N % k_per_group = 0
////////////////////////////////////////////////////////////////////////////////////
// 2 groups per CTA
default_multiple_group_sizes.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, threadblock_k * 4}, // input size (NHWC)
{threadblock_n, 3, 3, threadblock_k * 2}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation,
1, // split_k_slices
2 // groups
));
// 2 groups per CTA and partial gemm_k
default_multiple_group_sizes.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, threadblock_k}, // input size (NHWC)
{threadblock_n, 3, 3, threadblock_k / 2}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation,
1, // split_k_slices
2 // groups
));
// 4 groups per CTA
default_multiple_group_sizes.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, threadblock_k * 8}, // input size (NHWC)
{threadblock_n / 2, 3, 3, threadblock_k * 2}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation,
1, // split_k_slices
4 // groups
));
// 4 groups per CTA and partial gemm_k
default_multiple_group_sizes.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, threadblock_k * 2}, // input size (NHWC)
{threadblock_n / 2, 3, 3, threadblock_k / 2}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation,
1, // split_k_slices
4 // groups
));
}
};
} // namespace device
} // namespace conv
} // namespace test

7
test/unit/conv/device/conv2d_strided_dgrad_implicit_gemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32_sm80.cu
View File

@@ -85,7 +85,7 @@ TEST(SM80_Device_Conv2d_Strided_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32n
test::conv::device::Conv2dProblemVector problem_size_list;
#if 0 // run specific problem size in the unit test first
// run specific problem size in the unit test first
problem_size_list.push_back(cutlass::conv::Conv2dProblemSize(
{1, 4, 4, 8}, // input size (NHWC)
{8, 3, 3, 8}, // filter size (KRSC)
@@ -93,7 +93,6 @@ TEST(SM80_Device_Conv2d_Strided_Dgrad_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f32n
{3, 3}, // stride (stride_h, stride_w)
{1, 1} // dilation (dilation_h, dilation_w)
));
#endif
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>(problem_size_list));
@@ -281,7 +280,7 @@ TEST(SM80_Device_Conv2d_Strided_Dgrad_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32
test::conv::device::Conv2dProblemVector problem_size_list;
#if 0 // run specific problem size in the unit test first
// run specific problem size in the unit test first
problem_size_list.push_back(cutlass::conv::Conv2dProblemSize(
{1, 56, 56, 8}, // input size (NHWC)
{8, 1, 1, 8}, // filter size (KRSC)
@@ -298,8 +297,6 @@ TEST(SM80_Device_Conv2d_Strided_Dgrad_Optimized_ImplicitGemm_f16nhwc_f16nhwc_f32
{1, 1} // dilation (dilation_h, dilation_w)
));
#endif
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>(problem_size_list));
}

112
test/unit/conv/device/conv2d_strided_dgrad_implicit_gemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f32_sm80.cu Normal file
View File

@@ -0,0 +1,112 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_conv2d_dgrad.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Conv2d_Strided_Dgrad_Optimized_ImplicitGemm_tf32nhwc_tf32nhwc_f32nhwc_tensor_op_f32_align4,
64x64_32x5_32x32x32) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::tfloat32_t;
using ElementB = cutlass::tfloat32_t;
using ElementC = float;
using ElementAccumulator = float;
using ElementCompute = float;
/// Device-level Conv2d instance
using Conv2dDgradKernel = typename cutlass::conv::kernel::DefaultConv2dDgrad<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<16, 8, 8>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
4,
ElementAccumulator,
ElementCompute
>,
cutlass::conv::threadblock::StridedDgradIdentityThreadblockSwizzle<>,
5,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic,
cutlass::conv::StrideSupport::kStrided,
4,
4
>::Kernel;
using Conv2dDgrad = cutlass::conv::device::ImplicitGemmConvolution<Conv2dDgradKernel>;
test::conv::device::Conv2dProblemVector problem_size_list;
// run specific problem size in the unit test first
problem_size_list.push_back(cutlass::conv::Conv2dProblemSize(
{1, 1, 1, 16}, // input size (NHWC)
{8, 3, 3, 16}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{2, 1}, // stride (stride_h, stride_w)
{1, 1} // dilation (dilation_h, dilation_w)
));
// run specific problem size in the unit test first
problem_size_list.push_back(cutlass::conv::Conv2dProblemSize(
{1, 1, 1, 16}, // input size (NHWC)
{8, 3, 3, 16}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{3, 3}, // stride (stride_h, stride_w)
{1, 1} // dilation (dilation_h, dilation_w)
));
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestAllConv2d<Conv2dDgrad>(problem_size_list));
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED

4
test/unit/conv/device/conv2d_testbed.h
View File

@@ -602,7 +602,7 @@ bool TestAllConv2d(
conv_test_sizes, // run user specified sizes
conv_problems.conv2d_default_sizes, // run default and cudnn bug sizes
//conv_problems.conv2d_resnet50_sizes, // run resnet50 sizes
#if CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED
#if CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED
conv_problems.conv2d_rigorous_sizes, // run large and rigorous sizes if enabled
#endif
};
@@ -716,7 +716,7 @@ bool TestAllConv2d(
return true;
}
// CUTLASS DGRAD's *strided* specialization does not support split-k mode
if ((ImplicitGemm::kConvolutionalOperator ==
cutlass::conv::Operator::kDgrad) &&

221
test/unit/conv/device/depthwise_fprop_implicit_gemm_f16nhwc_f16nhwc_f16nhwc_simt_f16_sm60.cu Normal file
View File

@@ -0,0 +1,221 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_depthwise_fprop.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_testbed.h"
std::vector<cutlass::conv::Conv2dProblemSize> DepthwiseFpropProblemSizes() {
std::vector<cutlass::conv::Conv2dProblemSize> problems;
for ( int channels = 16; channels < 256 ; channels+=16){
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 8, 8, channels}, // input size (NHWC)
{channels, 3, 3, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{2, 2}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 16, 16, channels}, // input size (NHWC)
{channels, 3, 3, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{2, 2}, // stride (stride_h, stride_w)
{2, 2}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 16, 16, channels}, // input size (NHWC)
{channels, 7, 7, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 112, 112, channels}, // input size (NHWC)
{channels, 7, 7, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 112, 112, channels}, // input size (NHWC)
{channels, 7, 7, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{2, 2}, // stride (stride_h, stride_w)
{2, 2} , // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 112, 112, channels}, // input size (NHWC)
{channels, 5, 5, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{1, 1}, // stride (stride_h, stride_w)
{1, 1}, // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
problems.push_back(cutlass::conv::Conv2dProblemSize(
{1, 112, 112, channels}, // input size (NHWC)
{channels, 5, 5, 1}, // filter size (KRSC)
{1, 1, 1, 1}, // padding (pad_h, _, pad_w, _)
{2, 2}, // stride (stride_h, stride_w)
{2, 2} , // dilation (dilation_h, dilation_w)
cutlass::conv::Mode::kCrossCorrelation, // Convolution mode
1, // split_k_slices
channels // groups
));
}
return problems;
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM60_Device_Depthwise_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_simt_f16,
128x128_8x2_64x64x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using ElementCompute = cutlass::half_t;
/// Device-level depthwiseFpropKernel instance
using depthwiseFpropKernel = typename cutlass::conv::kernel::DefaultDepthwiseFprop<
ElementA,
cutlass::layout::TensorNHWC,
ElementB,
cutlass::layout::TensorNHWC,
ElementC,
cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm60,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<64, 64, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
2,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic
>::Kernel;
using DepthwiseFprop = cutlass::conv::device::ImplicitGemmConvolution<depthwiseFpropKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<DepthwiseFprop>(
DepthwiseFpropProblemSizes()));
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM60_Device_Depthwise_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_simt_f16,
64x64_8x2_32x32x8) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using ElementCompute = cutlass::half_t;
/// Device-level depthwiseFpropKernel instance
using depthwiseFpropKernel = typename cutlass::conv::kernel::DefaultDepthwiseFprop<
ElementA,
cutlass::layout::TensorNHWC,
ElementB,
cutlass::layout::TensorNHWC,
ElementC,
cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm60,
cutlass::gemm::GemmShape<64, 64, 8>,
cutlass::gemm::GemmShape<32, 32, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementC,
1,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
2,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::IteratorAlgorithm::kAnalytic
>::Kernel;
using DepthwiseFprop = cutlass::conv::device::ImplicitGemmConvolution<depthwiseFpropKernel>;
/// Run all unit test sizes with device-level Conv2d instance
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<DepthwiseFprop>(
DepthwiseFpropProblemSizes()));
}

246
test/unit/conv/device/group_conv2d_fprop_implicit_gemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32_sm80.cu Normal file
View File

@@ -0,0 +1,246 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Implicit GEMM interface
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/conv/kernel/default_conv2d_group_fprop.h"
#include "cutlass/conv/device/implicit_gemm_convolution.h"
#include "conv2d_testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Conv2d_Group_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32,
SingleGroupPerCTA_128x128_64x3_64x64x64) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
using ElementAccumulator = float;
using ElementCompute = float;
using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 64>;
using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 16>;
/// Device-level Conv2d instance
using Conv2dGroupFpropKernel = typename cutlass::conv::kernel::DefaultConv2dGroupFprop<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
ThreadblockShape,
WarpShape,
InstructionShape,
cutlass::epilogue::thread::LinearCombination<
ElementC,
128 / cutlass::sizeof_bits<ElementC>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::GroupMode::kSingleGroup,
cutlass::conv::IteratorAlgorithm::kAnalytic
>::Kernel;
using Conv2dGroupFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dGroupFpropKernel>;
/// Run group conv unit test sizes with device-level Conv2d instance
test::conv::device::TestbedGroupConv2dProblemSizes problem_sizes(
ThreadblockShape::kN, ThreadblockShape::kK,
128/cutlass::sizeof_bits<ElementA>::value
);
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<Conv2dGroupFprop>(problem_sizes.default_single_group_sizes));
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Conv2d_Group_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32,
SingleGroupPerCTA_64x64_64x3_32x32x64) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
using ElementAccumulator = float;
using ElementCompute = float;
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 64>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 16>;
/// Device-level Conv2d instance
using Conv2dGroupFpropKernel = typename cutlass::conv::kernel::DefaultConv2dGroupFprop<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
ThreadblockShape,
WarpShape,
InstructionShape,
cutlass::epilogue::thread::LinearCombination<
ElementC,
128 / cutlass::sizeof_bits<ElementC>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::GroupMode::kSingleGroup,
cutlass::conv::IteratorAlgorithm::kAnalytic
>::Kernel;
using Conv2dGroupFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dGroupFpropKernel>;
/// Run group conv unit test sizes with device-level Conv2d instance
test::conv::device::TestbedGroupConv2dProblemSizes problem_sizes(
ThreadblockShape::kN, ThreadblockShape::kK,
128/cutlass::sizeof_bits<ElementA>::value
);
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<Conv2dGroupFprop>(problem_sizes.default_single_group_sizes));
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Conv2d_Group_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32,
MultipleGroupPerCTA_128x128_64x3_64x64x64) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
using ElementAccumulator = float;
using ElementCompute = float;
using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 64>;
using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 16>;
/// Device-level Conv2d instance
using Conv2dGroupFpropKernel = typename cutlass::conv::kernel::DefaultConv2dGroupFprop<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
ThreadblockShape,
WarpShape,
InstructionShape,
cutlass::epilogue::thread::LinearCombination<
ElementC,
128 / cutlass::sizeof_bits<ElementC>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::GroupMode::kMultipleGroup,
cutlass::conv::IteratorAlgorithm::kAnalytic
>::Kernel;
using Conv2dGroupFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dGroupFpropKernel>;
/// Run group conv unit test sizes with device-level Conv2d instance
test::conv::device::TestbedGroupConv2dProblemSizes problem_sizes(
ThreadblockShape::kN, ThreadblockShape::kK,
128/cutlass::sizeof_bits<ElementA>::value
);
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<Conv2dGroupFprop>(problem_sizes.default_multiple_group_sizes));
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Conv2d_Group_Fprop_Analytic_ImplicitGemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f32,
MutipleGroupPerCTA_64x64_64x3_32x32x64) {
/// Conv operation element types for the Gemm equivalent (ImplicitGemm)
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementC = cutlass::half_t;
using ElementAccumulator = float;
using ElementCompute = float;
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 64>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 16>;
/// Device-level Conv2d instance
using Conv2dGroupFpropKernel = typename cutlass::conv::kernel::DefaultConv2dGroupFprop<
ElementA, cutlass::layout::TensorNHWC,
ElementB, cutlass::layout::TensorNHWC,
ElementC, cutlass::layout::TensorNHWC,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
ThreadblockShape,
WarpShape,
InstructionShape,
cutlass::epilogue::thread::LinearCombination<
ElementC,
128 / cutlass::sizeof_bits<ElementC>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3,
cutlass::arch::OpMultiplyAdd,
cutlass::conv::GroupMode::kMultipleGroup,
cutlass::conv::IteratorAlgorithm::kAnalytic
>::Kernel;
using Conv2dGroupFprop = cutlass::conv::device::ImplicitGemmConvolution<Conv2dGroupFpropKernel>;
/// Run group conv unit test sizes with device-level Conv2d instance
test::conv::device::TestbedGroupConv2dProblemSizes problem_sizes(
ThreadblockShape::kN, ThreadblockShape::kK,
128/cutlass::sizeof_bits<ElementA>::value
);
EXPECT_TRUE(test::conv::device::TestSpecificConv2d<Conv2dGroupFprop>(problem_sizes.default_multiple_group_sizes));
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_MMA_SM80_SUPPORTED
////////////////////////////////////////////////////////////////////////////////

41
test/unit/gemm/device/CMakeLists.txt
View File

@@ -340,6 +340,24 @@ cutlass_test_unit_add_executable(
gemm_grouped_sm80.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_grouped_scheduler
BATCH_SOURCES ON
BATCH_SIZE 4
gemm_grouped_scheduler_sm80.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_grouped_rank_2k_scheduler
BATCH_SOURCES ON
BATCH_SIZE 4
rank_2k_grouped_scheduler_sm80.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
@@ -540,4 +558,27 @@ cutlass_test_unit_add_executable(
hemm_cf32h_cf32n_tensor_op_fast_f32_rs_sm80.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_grouped_blas3
BATCH_SOURCES ON
BATCH_SIZE 4
# Grouped SYR2K SM80 f64 tests
syr2k_f64n_f64n_tensor_op_f64_grouped_sm80.cu
syr2k_f64n_f64t_tensor_op_f64_grouped_sm80.cu
syr2k_f64t_f64n_tensor_op_f64_grouped_sm80.cu
syr2k_f64t_f64t_tensor_op_f64_grouped_sm80.cu
# Grouped SYR2K SM80 cf64 tests
syr2k_cf64n_cf64n_tensor_op_f64_grouped_sm80.cu
syr2k_cf64n_cf64t_tensor_op_f64_grouped_sm80.cu
syr2k_cf64t_cf64n_tensor_op_f64_grouped_sm80.cu
syr2k_cf64t_cf64t_tensor_op_f64_grouped_sm80.cu
# Grouped HER2K SM80 f64 tests
her2k_cf64n_cf64n_tensor_op_f64_grouped_sm80.cu
her2k_cf64h_cf64n_tensor_op_f64_grouped_sm80.cu
)
endif()

222
test/unit/gemm/device/gemm_grouped_scheduler_sm80.cu Normal file
View File

@@ -0,0 +1,222 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped GEMM problem visitors
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/gemm_grouped.h"
#include "cutlass/gemm/kernel/default_gemm_grouped.h"
#include "cutlass/gemm/device/gemm_grouped.h"
#include "testbed_grouped_scheduler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
// Run a series of tests on the testbed
template <typename Testbed>
void run_tests() {
for (int scale_factor : {8, 16, 32, 64}) {
for (int threadblock_count : {54, 108, 216, 324, 432}) {
for (int problems : {1, 27, 180, 300}) {
Testbed testbed;
testbed.run(problems, threadblock_count, scale_factor);
}
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p128_t128, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 128;
static int const kThreadCount = 128;
static bool const kTranspose = false;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p128_t128_transpose, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 128;
static int const kThreadCount = 128;
static bool const kTranspose = true;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p256_t256, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static bool const kTranspose = false;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p256_t128, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 128;
static bool const kTranspose = false;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p256_t256, 64x32x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static bool const kTranspose = false;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p256_t256_transpose, 64x32x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static bool const kTranspose = true;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p256_t256, 32x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static bool const kTranspose = false;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGroupedScheduler_p256_t256_transpose, 32x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static bool const kTranspose = true;
using Testbed = test::gemm::device::TestbedGroupedGemmScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kTranspose,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

32
test/unit/gemm/device/gemm_grouped_sm80.cu
View File

@@ -181,7 +181,7 @@ struct GemmGroupedProblemVisitor {
}
CUTLASS_HOST_DEVICE
int64_t threadblock_index() const {
int64_t threadblock_idx() const {
return tile_idx - problem_tile_start;
}
@@ -193,7 +193,7 @@ struct GemmGroupedProblemVisitor {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <int CtaShapeM, int CtaShapeN>
template <int ThreadblockShapeM, int ThreadblockShapeN>
__global__ void GroupedBatchedKernel(GemmGroupedProblemVisitor::Params params) {
__shared__ GemmGroupedProblemVisitor::SharedStorage shared_storage;
@@ -201,18 +201,18 @@ __global__ void GroupedBatchedKernel(GemmGroupedProblemVisitor::Params params) {
GemmGroupedProblemVisitor problem_visitor(
shared_storage,
params,
{CtaShapeM, CtaShapeN},
{ThreadblockShapeM, ThreadblockShapeN},
blockIdx.x);
while (problem_visitor.next_tile()) {
cutlass::gemm::GemmCoord problem_size = problem_visitor.problem_size();
int64_t cta_idx = problem_visitor.threadblock_index();
int64_t threadblock_idx = problem_visitor.threadblock_idx();
cutlass::gemm::GemmCoord grid_shape = problem_visitor.grid_shape(problem_size);
int cta_tile_m_idx = int(cta_idx / grid_shape.n());
int cta_tile_n_idx = int(cta_idx % grid_shape.n());
int threadblock_tile_m_idx = int(threadblock_idx / grid_shape.n());
int threadblock_tile_n_idx = int(threadblock_idx % grid_shape.n());
//
// Do the MMA
@@ -220,13 +220,13 @@ __global__ void GroupedBatchedKernel(GemmGroupedProblemVisitor::Params params) {
if (threadIdx.x == 0) {
#if 0
printf("Block %d - tile: %lld, problem %d, cta_idx: %lld, cta(m: %d, n: %d)\n",
printf("Block %d - tile: %lld, problem %d, threadblock_idx: %lld, threadblock(m: %d, n: %d)\n",
blockIdx.x,
problem_visitor.tile_index(),
problem_visitor.problem_index(),
cta_idx,
cta_tile_m_idx,
cta_tile_n_idx);
threadblock_idx,
threadblock_tile_m_idx,
threadblock_tile_n_idx);
#endif
}
@@ -241,8 +241,8 @@ TEST(SM80_Device_GemmGrouped_scheduler, 64x64x32_32x32x32) {
int32_t problem_count = 16;
int const kCtaShapeM = 64;
int const kCtaShapeN = 64;
int const kThreadblockShapeM = 64;
int const kThreadblockShapeN = 64;
std::vector<cutlass::gemm::GemmCoord> problem_sizes(problem_count);
std::vector<int64_t> tile_counts(problem_count);
@@ -262,7 +262,7 @@ TEST(SM80_Device_GemmGrouped_scheduler, 64x64x32_32x32x32) {
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord grid_shape = GemmGroupedProblemVisitor::grid_shape(
problem_sizes.at(i), {kCtaShapeM, kCtaShapeN});
problem_sizes.at(i), {kThreadblockShapeM, kThreadblockShapeN});
int32_t problem_tile_count = (grid_shape.m() * grid_shape.n());
@@ -295,7 +295,7 @@ TEST(SM80_Device_GemmGrouped_scheduler, 64x64x32_32x32x32) {
dim3 grid(108, 1, 1);
dim3 block(128, 1, 1);
GroupedBatchedKernel<kCtaShapeM, kCtaShapeN><<< grid, block >>>(params);
GroupedBatchedKernel<kThreadblockShapeM, kThreadblockShapeN><<< grid, block >>>(params);
// wait
cudaDeviceSynchronize();
@@ -705,6 +705,7 @@ TEST(SM80_Device_GemmGrouped_cf32n_cf32n_cf32n_tensorop_f32, 64x64x16_32x32x16)
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
@@ -748,6 +749,7 @@ TEST(SM80_Device_GemmGrouped_cf32c_cf32t_cf32n_tensorop_f32, 64x64x16_32x32x16)
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
@@ -791,6 +793,7 @@ TEST(SM80_Device_GemmGrouped_cf32c_cf32t_cf32t_tensorop_f32, 64x64x16_32x32x16)
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
@@ -834,6 +837,7 @@ TEST(SM80_Device_GemmGrouped_cf32t_cf32h_cf32n_tensorop_f32, 64x64x16_16x16x16)
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;

2
test/unit/gemm/device/gemm_universal_f16n_f16t_f32t_tensor_op_f32_sm75.cu
View File

@@ -79,7 +79,6 @@ TEST(SM75_Device_GemmUniversal_f16n_f16t_f32t_tensor_op_f32, 64x64x32_32x32x32)
EXPECT_TRUE(test::gemm::device::TestAllGemmUniversal<Gemm>());
}
TEST(SM75_Device_GemmUniversal_f16n_f16t_f32t_tensor_op_f32, 64x64x32_32x32x32_updated_batch_count) {
using ElementOutput = float;
@@ -114,4 +113,3 @@ TEST(SM75_Device_GemmUniversal_f16n_f16t_f32t_tensor_op_f32, 64x64x32_32x32x32_u
#endif // #if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////

310
test/unit/gemm/device/her2k_cf64h_cf64n_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,310 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
// NOTE: HER2K requires that LayoutA == LayoutB, and that LayoutC == ColumnMajor
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_l_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_l_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_u_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64h_cf64n_u_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kConjugate, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kConjugate, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

310
test/unit/gemm/device/her2k_cf64n_cf64n_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,310 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
// NOTE: HER2K requires that LayoutA == LayoutB, and that LayoutC == ColumnMajor
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_l_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_l_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_u_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Her2KGrouped_cf64n_cf64n_u_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kHermitian>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

234
test/unit/gemm/device/rank_2k_grouped_scheduler_sm80.cu Normal file
View File

@@ -0,0 +1,234 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K problem visitors
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/gemm_grouped.h"
#include "cutlass/gemm/kernel/default_gemm_grouped.h"
#include "cutlass/gemm/device/gemm_grouped.h"
#include "testbed_grouped_rank_2k_scheduler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
// Run a series of tests on the testbed
template <typename Testbed>
void run_tests(bool skip_tile_check=false) {
for (int scale_factor : {8, 16, 32, 64}) {
for (int threadblock_count : {54, 108, 216, 324, 432}) {
for (int problems : {1, 27, 180, 300}) {
Testbed testbed(skip_tile_check);
testbed.run(problems, threadblock_count, scale_factor);
}
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p128_t128_l, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 128;
static int const kThreadCount = 128;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kLower;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p128_t128_u, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 128;
static int const kThreadCount = 128;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kUpper;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p256_t256_l, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kLower;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p256_t128_l, 64x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 128;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kLower;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
run_tests<Testbed>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p256_t256_l, 64x32x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kLower;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
// Skip individual tile check for the non-square SYR2K versions. We still
// compare the problem visitors with one another
run_tests<Testbed>(/*skip_tile_check=*/true);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p256_t256_u, 64x32x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kUpper;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
// Skip individual tile check for the non-square SYR2K versions. We still
// compare the problem visitors with one another
run_tests<Testbed>(/*skip_tile_check=*/true);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p256_t256_l, 32x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kLower;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
// Skip individual tile check for the non-square SYR2K versions. We still
// compare the problem visitors with one another
run_tests<Testbed>(/*skip_tile_check=*/true);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Rank2KGroupedScheduler_p256_t256_u, 32x64x32) {
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 32>;
static int const kNumPrefetch = 256;
static int const kThreadCount = 256;
static cutlass::FillMode const kFillModeC = cutlass::FillMode::kUpper;
using Testbed = test::gemm::device::TestbedGroupedRank2KScheduler<
ThreadblockShape,
kNumPrefetch,
kThreadCount,
kFillModeC,
// List of GroupScheduleModes to compare. List must contain at least two.
cutlass::gemm::kernel::GroupScheduleMode::kDeviceOnly,
cutlass::gemm::kernel::GroupScheduleMode::kHostPrecompute>;
// Skip individual tile check for the non-square SYR2K versions. We still
// compare the problem visitors with one another
run_tests<Testbed>(/*skip_tile_check=*/true);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

308
test/unit/gemm/device/syr2k_cf64n_cf64n_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,308 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_l_tensor_op_cf64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_l_tensor_op_cf64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_l_tensor_op_cf64, 64x32x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_l_tensor_op_cf64, 32x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_u_tensor_op_cf64, 32x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_u_tensor_op_cf64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_u_tensor_op_cf64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

168
test/unit/gemm/device/syr2k_cf64n_cf64t_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,168 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64t_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64t_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64t_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

168
test/unit/gemm/device/syr2k_cf64t_cf64n_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,168 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64n_cf64n_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

168
test/unit/gemm/device/syr2k_cf64t_cf64t_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,168 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64t_cf64t_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64t_cf64t_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_cf64t_cf64t_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = cutlass::complex<double>;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = cutlass::complex<double>;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = cutlass::complex<double>;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = cutlass::complex<double>;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAddComplex,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

483
test/unit/gemm/device/syr2k_f64n_f64n_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,483 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_l_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_l_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_l_tensor_op_f64, 128x64x16_64x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 64, 16>,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_l_tensor_op_f64, 128x128x16_32x64x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 16>,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_u_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_u_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_u_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_u_tensor_op_f64, 128x64x16_64x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 64, 16>,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64n_u_tensor_op_f64, 128x128x16_32x64x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 16>,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

273
test/unit/gemm/device/syr2k_f64n_f64t_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,273 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64t_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64t_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64t_l_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64t_l_tensor_op_f64, 128x64x16_64x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 64, 16>,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64t_l_tensor_op_f64, 128x128x16_32x64x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 16>,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64n_f64t_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = double;
using LayoutB = cutlass::layout::ColumnMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

308
test/unit/gemm/device/syr2k_f64t_f64n_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,308 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_l_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_l_tensor_op_f64, 128x64x16_64x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 64, 16>,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_l_tensor_op_f64, 128x128x16_32x64x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 16>,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64n_u_tensor_op_f64, 64x32x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::ColumnMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

308
test/unit/gemm/device/syr2k_f64t_f64t_tensor_op_f64_grouped_sm80.cu Normal file
View File

@@ -0,0 +1,308 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/blas3.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped_rank_2k.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_l_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_l_tensor_op_f64, 64x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_l_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_l_tensor_op_f64, 128x64x16_64x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 64, 16>,
cutlass::gemm::GemmShape<64, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_l_tensor_op_f64, 128x128x16_32x64x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kLower,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 16>,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_u_tensor_op_f64, 32x32x16_16x16x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_Syr2kGrouped_f64t_f64t_u_tensor_op_f64, 32x64x16_32x32x16) {
using ElementA = double;
using LayoutA = cutlass::layout::RowMajor;
using ElementB = double;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = double;
using LayoutC = cutlass::layout::RowMajor;
using ElementAccumulator = double;
using Rank2Kkernel = typename cutlass::gemm::kernel::DefaultRank2KGrouped<
ElementA, LayoutA, cutlass::ComplexTransform::kNone, 1,
ElementB, LayoutB, cutlass::ComplexTransform::kNone, 1,
ElementC, LayoutC, cutlass::FillMode::kUpper,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<ElementC, 1, ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
3, // kStages
cutlass::arch::OpMultiplyAdd,
cutlass::BlasMode::kSymmetric>::Rank2Kkernel;
using Rank2K = cutlass::gemm::device::Rank2KGrouped<Rank2Kkernel>;
test::gemm::device::TestbedGrouped<Rank2K> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////

47
test/unit/gemm/device/testbed_grouped.h
View File

@@ -417,46 +417,27 @@ struct TestbedGrouped {
return passed;
}
/// Returns the number of threadblocks to launch if the kernel can run on the target
/// device. Otherwise, returns zero.
int sufficient() const {
cudaDeviceProp properties;
int device_idx;
cudaError_t result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() API call failed.");
}
result = cudaGetDeviceProperties(&properties, device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed");
}
int occupancy = Gemm::maximum_active_blocks();
return properties.multiProcessorCount * occupancy;
}
/// Executes one test
bool run(
int problem_count,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(0)) {
int threadblock_count = sufficient();
// Early exit
if (!threadblock_count) {
return false;
}
this->problem_count = problem_count;
// Initialize the problem
initialize();
int threadblock_count = Gemm::sufficient(problem_sizes_host.data(), problem_count);
// Early exit
if (!threadblock_count) {
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
std::cerr << "Test waived due to insufficient CUDA device resources." << std::endl;
}
return true;
}
// Configure the GEMM arguments
typename EpilogueOutputOp::Params epilogue_op(alpha, beta);
@@ -473,13 +454,17 @@ struct TestbedGrouped {
lda.get(),
ldb.get(),
ldc.get(),
ldd.get()
ldd.get(),
problem_sizes_host.data()
);
// Initialize the GEMM object
Gemm gemm;
cutlass::Status status = gemm.initialize(args);
size_t workspace_size = gemm.get_workspace_size(args);
cutlass::DeviceAllocation<uint8_t> workspace(workspace_size);
cutlass::Status status = gemm.initialize(args, workspace.get());
if (status != cutlass::Status::kSuccess) {
return false;

502
test/unit/gemm/device/testbed_grouped_rank_2k.h Normal file
View File

@@ -0,0 +1,502 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K interface
*/
#pragma once
#include <fstream>
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/device_kernel.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped.h"
#include "cutlass/gemm/kernel/default_rank_2k_grouped.h"
#include "cutlass/gemm/device/rank_2k_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/rank_2k_complex.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/tensor_view_io.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Rank2K>
struct TestbedGrouped {
//
// Type definitions
//
using ElementA = typename Rank2K::ElementA;
using ElementB = typename Rank2K::ElementB;
using ElementC = typename Rank2K::ElementC;
using ElementAccumulator = typename Rank2K::ElementAccumulator;
using EpilogueOutputOp = typename Rank2K::EpilogueOutputOp;
using ElementCompute = typename EpilogueOutputOp::ElementCompute;
using LayoutA = typename Rank2K::LayoutA;
using LayoutB = typename Rank2K::LayoutB;
using LayoutC = typename Rank2K::LayoutC;
using MatrixCoord = typename LayoutC::TensorCoord;
//
// Data members
//
/// Initialization
cutlass::Distribution::Kind init_A;
cutlass::Distribution::Kind init_B;
cutlass::Distribution::Kind init_C;
uint32_t seed;
int problem_count;
std::vector<cutlass::gemm::GemmCoord> problem_sizes_host;
cutlass::DeviceAllocation<cutlass::gemm::GemmCoord> problem_sizes_device;
std::vector<int64_t> offset_A;
std::vector<int64_t> offset_B;
std::vector<int64_t> offset_C;
std::vector<int64_t> offset_D;
std::vector<int64_t> lda_host;
std::vector<int64_t> ldb_host;
std::vector<int64_t> ldc_host;
std::vector<int64_t> ldd_host;
cutlass::DeviceAllocation<int64_t> lda;
cutlass::DeviceAllocation<int64_t> ldb;
cutlass::DeviceAllocation<int64_t> ldc;
cutlass::DeviceAllocation<int64_t> ldd;
cutlass::DeviceAllocation<ElementA> block_A;
cutlass::DeviceAllocation<ElementB> block_B;
cutlass::DeviceAllocation<ElementC> block_C;
cutlass::DeviceAllocation<ElementC> block_D;
cutlass::DeviceAllocation<ElementA *> ptr_A;
cutlass::DeviceAllocation<ElementB *> ptr_B;
cutlass::DeviceAllocation<ElementC *> ptr_C;
cutlass::DeviceAllocation<ElementC *> ptr_D;
//
// Methods
//
TestbedGrouped(
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
uint32_t seed_ = 3080
):
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
/// Helper to initialize a tensor view
template <typename Element, typename Layout>
bool initialize_tensor(
cutlass::TensorView<Element, Layout> view,
cutlass::Distribution::Kind dist_kind,
uint32_t seed) {
if (dist_kind == cutlass::Distribution::Uniform) {
double scope_max, scope_min;
int bits_input = cutlass::sizeof_bits<Element>::value;
int bits_output = cutlass::sizeof_bits<typename Rank2K::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
scope_min = -2;
} else if (bits_output == 16) {
if (cutlass::sizeof_bits<ElementAccumulator>::value <= 16) {
scope_max = 5;
scope_min = -5;
}
else {
scope_max = 8;
scope_min = -8;
}
} else {
scope_max = 8;
scope_min = -8;
}
cutlass::reference::host::TensorFillRandomUniform(
view, seed, scope_max, scope_min, 0);
}
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
}
else if (dist_kind == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(
view.data(), view.capacity());
}
else {
// no fill - remain zero
}
return true;
}
/// Initializes data structures
void initialize() {
//
// Choose random problem sizes
//
// construct a few problems of random sizes
srand(seed);
int64_t total_elements_A = 0;
int64_t total_elements_B = 0;
int64_t total_elements_C = 0;
int64_t total_elements_D = 0;
lda_host.resize(problem_count);
ldb_host.resize(problem_count);
ldc_host.resize(problem_count);
ldd_host.resize(problem_count);
problem_sizes_host.clear();
problem_sizes_host.resize(problem_count);
for (int32_t i = 0; i < problem_count; ++i) {
auto N = 8 * (rand() % 64) + 24;
auto K = 8 * (rand() % 64) + 24;
cutlass::gemm::GemmCoord problem(N, N, K);
if (!i) {
problem = cutlass::gemm::GemmCoord(16, 16, 8);
}
problem_sizes_host.at(i) = problem;
lda_host.at(i) = LayoutA::packed({problem.n(), problem.k()}).stride(0);
ldb_host.at(i) = LayoutB::packed({problem.n(), problem.k()}).stride(0);
ldc_host.at(i) = LayoutC::packed({problem.n(), problem.n()}).stride(0);
ldd_host.at(i) = LayoutC::packed({problem.n(), problem.n()}).stride(0);
offset_A.push_back(total_elements_A);
offset_B.push_back(total_elements_B);
offset_C.push_back(total_elements_C);
offset_D.push_back(total_elements_D);
int64_t elements_A = problem.n() * problem.k();
int64_t elements_B = problem.n() * problem.k();
int64_t elements_C = problem.n() * problem.n();
int64_t elements_D = problem.n() * problem.n();
total_elements_A += elements_A;
total_elements_B += elements_B;
total_elements_C += elements_C;
total_elements_D += elements_D;
// Random strides between problems?
}
problem_sizes_device.reset(problem_count);
problem_sizes_device.copy_from_host(problem_sizes_host.data());
lda.reset(problem_count);
ldb.reset(problem_count);
ldc.reset(problem_count);
ldd.reset(problem_count);
lda.copy_from_host(lda_host.data());
ldb.copy_from_host(ldb_host.data());
ldc.copy_from_host(ldc_host.data());
ldd.copy_from_host(ldd_host.data());
//
// Assign pointers
//
block_A.reset(total_elements_A);
block_B.reset(total_elements_B);
block_C.reset(total_elements_C);
block_D.reset(total_elements_D);
std::vector<ElementA *> ptr_A_host(problem_count);
std::vector<ElementB *> ptr_B_host(problem_count);
std::vector<ElementC *> ptr_C_host(problem_count);
std::vector<ElementC *> ptr_D_host(problem_count);
for (int32_t i = 0; i < problem_count; ++i) {
ptr_A_host.at(i) = block_A.get() + offset_A.at(i);
ptr_B_host.at(i) = block_B.get() + offset_B.at(i);
ptr_C_host.at(i) = block_C.get() + offset_C.at(i);
ptr_D_host.at(i) = block_D.get() + offset_D.at(i);
}
ptr_A.reset(problem_count);
ptr_A.copy_from_host(ptr_A_host.data());
ptr_B.reset(problem_count);
ptr_B.copy_from_host(ptr_B_host.data());
ptr_C.reset(problem_count);
ptr_C.copy_from_host(ptr_C_host.data());
ptr_D.reset(problem_count);
ptr_D.copy_from_host(ptr_D_host.data());
//
// Initialize the problems of the workspace
//
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord problem = problem_sizes_host.at(i);
LayoutA layout_A(lda_host.at(i));
LayoutB layout_B(ldb_host.at(i));
LayoutC layout_C(ldc_host.at(i));
LayoutC layout_D(ldd_host.at(i));
MatrixCoord extent_A{problem.n(), problem.k()};
MatrixCoord extent_B{problem.n(), problem.k()};
MatrixCoord extent_C{problem.n(), problem.n()};
std::vector<ElementA> matrix_A(layout_A.capacity(extent_A));
std::vector<ElementB> matrix_B(layout_B.capacity(extent_B));
std::vector<ElementC> matrix_C(layout_C.capacity(extent_C));
std::vector<ElementC> matrix_D(layout_D.capacity(extent_C));
initialize_tensor(cutlass::TensorView<ElementA, LayoutA>(matrix_A.data(), layout_A, extent_A), init_A, seed * 2021);
initialize_tensor(cutlass::TensorView<ElementB, LayoutB>(matrix_B.data(), layout_B, extent_B), init_B, seed * 2022);
initialize_tensor(cutlass::TensorView<ElementC, LayoutC>(matrix_C.data(), layout_C, extent_C), init_C, seed * 2023);
cutlass::device_memory::copy_to_device(ptr_A_host.at(i), matrix_A.data(), matrix_A.size());
cutlass::device_memory::copy_to_device(ptr_B_host.at(i), matrix_B.data(), matrix_B.size());
cutlass::device_memory::copy_to_device(ptr_C_host.at(i), matrix_C.data(), matrix_C.size());
cutlass::device_memory::copy_to_device(ptr_D_host.at(i), matrix_D.data(), matrix_D.size());
}
}
/// Verifies the result is a Rank2K
bool verify(
ElementCompute alpha,
ElementCompute beta) {
bool passed = true;
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord problem = problem_sizes_host.at(i);
LayoutA layout_A(lda_host.at(i));
LayoutB layout_B(ldb_host.at(i));
LayoutC layout_C(ldc_host.at(i));
LayoutC layout_D(ldd_host.at(i));
MatrixCoord extent_A{problem.n(), problem.k()};
MatrixCoord extent_B{problem.n(), problem.k()};
MatrixCoord extent_C{problem.n(), problem.n()};
std::vector<ElementA> matrix_A(layout_A.capacity(extent_A));
std::vector<ElementB> matrix_B(layout_B.capacity(extent_B));
std::vector<ElementC> matrix_C(layout_C.capacity(extent_C));
std::vector<ElementC> matrix_D(layout_D.capacity(extent_C));
std::vector<ElementC> matrix_Ref(layout_D.capacity(extent_C));
cutlass::device_memory::copy_to_host(matrix_A.data(), block_A.get() + offset_A.at(i), matrix_A.size());
cutlass::device_memory::copy_to_host(matrix_B.data(), block_B.get() + offset_B.at(i), matrix_B.size());
cutlass::device_memory::copy_to_host(matrix_C.data(), block_C.get() + offset_C.at(i), matrix_C.size());
cutlass::device_memory::copy_to_host(matrix_D.data(), block_D.get() + offset_D.at(i), matrix_D.size());
cutlass::TensorView<ElementA, LayoutA> view_A(matrix_A.data(), layout_A, extent_A);
cutlass::TensorView<ElementB, LayoutB> view_B(matrix_B.data(), layout_B, extent_B);
cutlass::TensorView<ElementC, LayoutC> view_C(matrix_C.data(), layout_C, extent_C);
cutlass::TensorView<ElementC, LayoutC> view_D(matrix_D.data(), layout_D, extent_C);
cutlass::TensorView<ElementC, LayoutC> view_Ref(matrix_Ref.data(), layout_D, extent_C);
// Reference Rank2K
cutlass::reference::host::Rank2KComplex<
ElementA, LayoutA,
ElementB, LayoutB,
ElementC, LayoutC,
ElementCompute, ElementAccumulator
>(
problem,
alpha,
view_A,
Rank2K::kTransformA,
view_B,
Rank2K::kTransformB,
beta,
view_C,
view_Ref,
ElementAccumulator(0),
Rank2K::kFillModeC,
Rank2K::kBlasMode
);
// Ensure that no input or output is entirely zero
EXPECT_GT(cutlass::reference::host::TensorNorm(view_A), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_B), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_C), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_D), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_Ref), 0);
// Compare against reference
passed = cutlass::reference::host::TensorEquals(view_D, view_Ref);
if (!passed) {
std::ofstream file("testbed_grouped_errors.txt");
file
<< "problem: " << problem << " [group: " << i << "]\n"
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
file
<< "A =\n" << view_A
<< "\nB =\n" << view_B
<< "\nC =\n" << view_C
<< "\n\nReference =\n" << view_Ref
<< "\nComputed =\n" << view_D;
return passed;
}
}
return passed;
}
/// Executes one test
bool run(
int problem_count,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(0)) {
this->problem_count = problem_count;
// Initialize the problem
initialize();
int threadblock_count = Rank2K::sufficient(problem_sizes_host.data(), problem_count);
// Early exit
if (!threadblock_count) {
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
std::cerr << "Test waived due to insufficient CUDA device resources." << std::endl;
}
return true;
}
// Configure the Rank2K arguments
typename EpilogueOutputOp::Params epilogue_op(alpha, beta);
// Configure Rank2K arguments
typename Rank2K::Arguments args(
cutlass::gemm::GemmUniversalMode::kGemm,
problem_sizes_device.get(),
problem_count,
threadblock_count,
epilogue_op,
ptr_A.get(),
ptr_B.get(),
ptr_C.get(),
ptr_D.get(),
lda.get(),
ldb.get(),
ldc.get(),
ldd.get(),
problem_sizes_host.data()
);
// Initialize the Rank2K object
Rank2K rank2k;
size_t workspace_size = rank2k.get_workspace_size(args);
cutlass::DeviceAllocation<uint8_t> workspace(workspace_size);
cutlass::Status status = rank2k.initialize(args, workspace.get());
if (status != cutlass::Status::kSuccess) {
return false;
}
// Run the Rank2K object
status = rank2k.run();
if (status != cutlass::Status::kSuccess) {
return false;
}
// Wait for completion
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess)
<< "Kernel execution error: " << cudaGetErrorString(result);
if (result != cudaSuccess) {
return false;
}
// Verify correctness
return verify(alpha, beta);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // device
} // gemm
} // test
/////////////////////////////////////////////////////////////////////////////////////////////////

461
test/unit/gemm/device/testbed_grouped_rank_2k_scheduler.h Normal file
View File

@@ -0,0 +1,461 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped Rank2K problem visitors
*/
#pragma once
#include <iostream>
#include <numeric>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/rank_2k_grouped_problem_visitor.h"
#include "cutlass/util/device_memory.h"
#include "cutlass/device_kernel.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
// Use simple problem visitor as a baseline
template <typename ProblemSizeHelper,
typename ThreadblockShape,
int PrefetchTileCount,
int ThreadCount,
cutlass::FillMode FillModeC>
struct BaselineProblemVisitor : public cutlass::gemm::kernel::BaseGroupedProblemVisitor<ProblemSizeHelper, ThreadblockShape> {
using Base = cutlass::gemm::kernel::BaseGroupedProblemVisitor<ProblemSizeHelper, ThreadblockShape>;
using Params = typename Base::Params;
static int const kThreadCount = ThreadCount;
static cutlass::FillMode const kFillModeC = FillModeC;
struct SharedStorage {};
int32_t tile_count_sum;
SharedStorage &shared_storage;
//
// Methods
//
CUTLASS_DEVICE
BaselineProblemVisitor(
Params const &params_,
SharedStorage &shared_storage_,
int32_t block_idx
): Base(params_, block_idx),
shared_storage(shared_storage_)
{
cutlass::gemm::GemmCoord problem = this->problem_size();
cutlass::gemm::GemmCoord grid = this->grid_shape(problem);
tile_count_sum = this->tile_count(grid);
}
CUTLASS_DEVICE
bool next_tile() {
if (this->tile_idx < tile_count_sum) {
return true;
}
do {
++this->problem_idx;
if (this->problem_idx >= this->params.problem_count) {
return false;
}
cutlass::gemm::GemmCoord problem = this->problem_size();
cutlass::gemm::GemmCoord grid = this->grid_shape(problem);
this->problem_tile_start = tile_count_sum;
tile_count_sum += this->tile_count(grid);
} while (tile_count_sum <= this->tile_idx);
return true;
}
static size_t get_workspace_size(const cutlass::gemm::GemmCoord* host_problem_sizes_ptr,
int32_t problem_count,
int32_t block_count) {
return 0;
}
static void host_precompute(const cutlass::gemm::GemmCoord* host_problem_sizes_ptr,
int32_t problem_count,
int32_t block_count,
void* host_workspace_ptr) {}
CUTLASS_DEVICE
cutlass::gemm::GemmCoord threadblock_offset(int32_t threadblock_id) const {
int32_t macro_id = threadblock_id / ProblemSizeHelper::OffsetHelper::kThreadblockSkewRatio;
int32_t macro_row = ceil(cutlass::fast_sqrt((2*macro_id) + 2.25) - 0.5) - 1;
int32_t macro_col = macro_id - (((macro_row+1) * macro_row)/2);
if (FillModeC == cutlass::FillMode::kUpper) {
cutlass::swap(macro_row, macro_col);
}
int32_t row = ProblemSizeHelper::OffsetHelper::macro_row_to_row(macro_row, threadblock_id);
int32_t col = ProblemSizeHelper::OffsetHelper::macro_col_to_col(macro_col, threadblock_id);
return cutlass::gemm::GemmCoord(row, col, 0);
}
};
template <typename ProblemVisitor>
struct ProblemVisitorKernel {
struct SharedStorage {
typename ProblemVisitor::SharedStorage problem_visitor;
};
struct Params {
typename ProblemVisitor::Params problem_visitor_params;
int32_t* visited_problems_ptr;
int32_t* visited_tiles_ptr;
int32_t visits_per_block;
Params():
visited_problems_ptr(nullptr),
visited_tiles_ptr(nullptr),
visits_per_block(0) {}
Params(typename ProblemVisitor::Params problem_visitor_params_,
int32_t* visited_problems_ptr_,
int32_t* visited_tiles_ptr_,
int32_t visits_per_block_):
problem_visitor_params(problem_visitor_params_),
visited_problems_ptr(visited_problems_ptr_),
visited_tiles_ptr(visited_tiles_ptr_),
visits_per_block(visits_per_block_) {}
};
CUTLASS_DEVICE
void operator()(const Params& params, SharedStorage &shared_storage) {
int32_t store_offset = params.visits_per_block * blockIdx.x;
ProblemVisitor problem_visitor(params.problem_visitor_params,
shared_storage.problem_visitor,
blockIdx.x);
while (problem_visitor.next_tile()) {
cutlass::gemm::GemmCoord problem_size = problem_visitor.problem_size();
int32_t problem_idx = problem_visitor.problem_index();
int32_t threadblock_idx = int32_t(problem_visitor.threadblock_idx());
cutlass::gemm::GemmCoord grid_shape = problem_visitor.grid_shape(problem_size);
cutlass::gemm::GemmCoord tile_offset = problem_visitor.threadblock_offset(threadblock_idx);
problem_visitor.advance(gridDim.x);
//
// Early exit conditions
// 1) Out of range
// 2) Upper-triangular block in lower-triangular problem
// 3) Lower-triangular block in upper-triangular problem
//
if (grid_shape.m() <= tile_offset.m() ||
grid_shape.n() <= tile_offset.n()) {
continue;
}
if (ProblemVisitor::kFillModeC == cutlass::FillMode::kLower &&
(tile_offset.m() + 1) * ProblemVisitor::ThreadblockShape::kM <= tile_offset.n() * ProblemVisitor::ThreadblockShape::kN) {
continue;
}
if (ProblemVisitor::kFillModeC == cutlass::FillMode::kUpper &&
tile_offset.m() * ProblemVisitor::ThreadblockShape::kM >= (tile_offset.n() + 1) * ProblemVisitor::ThreadblockShape::kN) {
continue;
}
if (threadIdx.x == 0) {
params.visited_problems_ptr[store_offset] = problem_idx;
params.visited_tiles_ptr[store_offset] = threadblock_idx;
++store_offset;
}
}
}
};
template <typename ProblemVisitor>
struct ProblemVisitorRunner {
using BaseKernel = ProblemVisitorKernel<ProblemVisitor>;
using Params = typename BaseKernel::Params;
Params params;
std::vector<cutlass::gemm::GemmCoord> host_problem_sizes;
int32_t problem_count;
int32_t threadblock_count;
int32_t visits_per_block;
cutlass::DeviceAllocation<int32_t> visited_problems;
cutlass::DeviceAllocation<int32_t> visited_tiles;
cutlass::DeviceAllocation<cutlass::gemm::GemmCoord> device_problem_sizes;
cutlass::DeviceAllocation<uint8_t> workspace;
std::vector<int32_t> host_visited_problems;
std::vector<int32_t> host_visited_tiles;
ProblemVisitorRunner(const std::vector<cutlass::gemm::GemmCoord>& host_problem_sizes_,
int32_t threadblock_count_):
host_problem_sizes(host_problem_sizes_),
problem_count(int32_t(host_problem_sizes_.size())),
threadblock_count(threadblock_count_) {}
/// Initializes GEMM state from arguments.
cutlass::Status initialize() {
size_t workspace_bytes = ProblemVisitor::get_workspace_size(
host_problem_sizes.data(),
problem_count,
threadblock_count);
workspace.reset(workspace_bytes);
std::vector<uint8_t> host_workspace(workspace_bytes);
int32_t tile_count = ProblemVisitor::group_tile_count(host_problem_sizes.data(), problem_count);
ProblemVisitor::host_precompute(host_problem_sizes.data(), problem_count,
threadblock_count, host_workspace.data());
workspace.copy_from_host(host_workspace.data(), workspace_bytes);
device_problem_sizes.reset(problem_count);
device_problem_sizes.copy_from_host(host_problem_sizes.data(), problem_count);
visits_per_block = (tile_count - 1 + threadblock_count) / threadblock_count;
int32_t total_visits = visits_per_block * threadblock_count;
visited_problems.reset(total_visits);
visited_tiles.reset(total_visits);
host_visited_problems.resize(total_visits);
host_visited_tiles.resize(total_visits);
cudaError_t result = cudaMemset(visited_problems.get(), -1, sizeof(int32_t) * total_visits);
if (result != cudaSuccess) {
return cutlass::Status::kErrorInternal;
}
result = cudaMemset(visited_tiles.get(), -1, sizeof(int32_t) * total_visits);
if (result != cudaSuccess) {
return cutlass::Status::kErrorInternal;
}
typename ProblemVisitor::Params pv_params(device_problem_sizes.get(), problem_count, workspace.get(), tile_count);
params = Params(pv_params, visited_problems.get(), visited_tiles.get(), visits_per_block);
return cutlass::Status::kSuccess;
}
bool verify() {
// Sort by problem size and then by threadblock_idx
std::vector<int32_t> indices(host_visited_problems.size());
std::iota(indices.begin(), indices.end(), 0);
std::stable_sort(indices.begin(), indices.end(),
[&](int32_t i1, int32_t i2) {
if (host_visited_problems[i1] == host_visited_problems[i2]) {
return host_visited_tiles[i1] < host_visited_tiles[i2];
}
return host_visited_problems[i1] < host_visited_problems[i2];
});
int32_t idx = 0;
// Skip any entries that were not visited
while (host_visited_problems[indices[idx]] == -1) {
++idx;
}
// Check that each problem visited has the tiles we expect
for (int32_t problem_idx = 0; problem_idx < problem_count; ++problem_idx) {
auto problem = host_problem_sizes[problem_idx];
ProblemVisitor::possibly_transpose_problem(problem);
int32_t problem_tiles = ProblemVisitor::tile_count(ProblemVisitor::grid_shape(problem));
for (int i = 0; i < problem_tiles; ++i) {
EXPECT_EQ(problem_idx, host_visited_problems[indices[idx]]);
EXPECT_EQ(i, host_visited_tiles[indices[idx]]);
++idx;
}
}
return true;
}
bool run(bool skip_tile_check=false, cudaStream_t stream = nullptr) {
cutlass::Status status = initialize();
if (status != cutlass::Status::kSuccess) {
std::cerr << "Initialization failed" << std::endl;
return false;
}
dim3 grid(threadblock_count, 1, 1);
dim3 block(ProblemVisitor::kThreadCount, 1, 1);
int smem_size = int(sizeof(typename BaseKernel::SharedStorage));
cutlass::Kernel<BaseKernel><<<grid, block, smem_size, stream>>>(params);
cudaError_t result = cudaGetLastError();
if (result != cudaSuccess) {
std::cerr << "grid launch failed with error " << cudaGetErrorString(result) << std::endl;
return false;
}
result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
std::cerr << "cudaDeviceSynchronize failed with error " << cudaGetErrorString(result) << std::endl;
return false;
}
visited_problems.copy_to_host(host_visited_problems.data());
visited_tiles.copy_to_host(host_visited_tiles.data());
if (skip_tile_check) {
return true;
}
return verify();
}
};
template <typename ThreadblockShape,
int PrefetchTileCount,
int ThreadCount,
cutlass::FillMode FillModeC,
cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode0,
cutlass::gemm::kernel::GroupScheduleMode... Args>
struct TestbedGroupedRank2KScheduler {
using BaselinePV = BaselineProblemVisitor<cutlass::gemm::kernel::detail::Rank2KGroupedProblemSizeHelper<ThreadblockShape>,
ThreadblockShape,
PrefetchTileCount,
ThreadCount,
FillModeC>;
//
// Data members
//
// Whether to skip checking that the tiles are visited as expected. This is useful
// in cases where ThreadblockShape::kM != ThreadblockShape::kN, for which the grouped
// Rank2K scheduler may assign out-of-bounds tiles that will cause a threadblock to
// exit early, but which are difficult to detect in tests without reimplementing
// this functionality.
bool skip_tile_check;
uint32_t seed;
int problem_count;
int threadblock_count;
std::vector<cutlass::gemm::GemmCoord> problem_sizes_host;
//
// Methods
//
TestbedGroupedRank2KScheduler(bool skip_tile_check_=false, uint32_t seed_ = 3080):
skip_tile_check(skip_tile_check_), seed(seed_) { srand(seed); }
/// Initializes data structures
void initialize(int32_t scale_factor) {
//
// Choose random problem sizes
//
problem_sizes_host.clear();
problem_sizes_host.resize(problem_count);
for (int32_t i = 0; i < problem_count; ++i) {
int n = scale_factor * (rand() % 64) + 24;
cutlass::gemm::GemmCoord problem(
n,
n,
scale_factor * (rand() % 64) + 24);
problem_sizes_host.at(i) = problem;
}
}
template <cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode_>
void compare_visitors(const ProblemVisitorRunner<BaselinePV>& baseline_runner) {
using PV = cutlass::gemm::kernel::Rank2KGroupedProblemVisitor<
ThreadblockShape,
GroupScheduleMode_,
PrefetchTileCount,
ThreadCount,
FillModeC>;
ProblemVisitorRunner<PV> runner(problem_sizes_host, threadblock_count);
EXPECT_TRUE(runner.run(skip_tile_check));
// Check that this problem visitor visits the same problems and tiles as the baseline
EXPECT_EQ(baseline_runner.host_visited_problems, runner.host_visited_problems);
EXPECT_EQ(baseline_runner.host_visited_tiles, runner.host_visited_tiles);
}
template <cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode1_,
cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode2_,
cutlass::gemm::kernel::GroupScheduleMode... Rest>
void compare_visitors(const ProblemVisitorRunner<BaselinePV>& baseline_runner) {
// Compare the next visitor with the baseline visitor
compare_visitors<GroupScheduleMode1_>(baseline_runner);
// Recurse to compare the next visitors
compare_visitors<GroupScheduleMode2_, Rest...>(baseline_runner);
}
/// Executes the test on all scheduler modes
void run(int problem_count, int threadblock_count, int scale_factor=8) {
this->problem_count = problem_count;
this->threadblock_count = threadblock_count;
// Initialize the problem
initialize(scale_factor);
// Run the baseline visitor to which we will compare all other visitors
ProblemVisitorRunner<BaselinePV> baseline_runner(problem_sizes_host, threadblock_count);
EXPECT_TRUE(baseline_runner.run(skip_tile_check));
compare_visitors<Args...>(baseline_runner);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // device
} // gemm
} // test
/////////////////////////////////////////////////////////////////////////////////////////////////

406
test/unit/gemm/device/testbed_grouped_scheduler.h Normal file
View File

@@ -0,0 +1,406 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for grouped GEMM problem visitors
*/
#pragma once
#include <iostream>
#include <numeric>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/gemm_grouped_problem_visitor.h"
#include "cutlass/gemm/kernel/grouped_problem_visitor.h"
#include "cutlass/util/device_memory.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
// Use simple problem visitor as a baseline
template <typename ProblemSizeHelper,
typename ThreadblockShape,
int PrefetchTileCount,
int ThreadCount>
struct BaselineProblemVisitor : public cutlass::gemm::kernel::BaseGroupedProblemVisitor<ProblemSizeHelper, ThreadblockShape> {
using Base = cutlass::gemm::kernel::BaseGroupedProblemVisitor<ProblemSizeHelper, ThreadblockShape>;
using Params = typename Base::Params;
static int const kThreadCount = ThreadCount;
struct SharedStorage {};
int32_t tile_count_sum;
SharedStorage &shared_storage;
//
// Methods
//
CUTLASS_DEVICE
BaselineProblemVisitor(
Params const &params_,
SharedStorage &shared_storage_,
int32_t block_idx
): Base(params_, block_idx),
shared_storage(shared_storage_)
{
cutlass::gemm::GemmCoord problem = this->problem_size();
cutlass::gemm::GemmCoord grid = this->grid_shape(problem);
tile_count_sum = this->tile_count(grid);
}
CUTLASS_DEVICE
bool next_tile() {
if (this->tile_idx < tile_count_sum) {
return true;
}
do {
++this->problem_idx;
if (this->problem_idx >= this->params.problem_count) {
return false;
}
cutlass::gemm::GemmCoord problem = this->problem_size();
cutlass::gemm::GemmCoord grid = this->grid_shape(problem);
this->problem_tile_start = tile_count_sum;
tile_count_sum += this->tile_count(grid);
} while (tile_count_sum <= this->tile_idx);
return true;
}
static size_t get_workspace_size(const cutlass::gemm::GemmCoord* host_problem_sizes_ptr,
int32_t problem_count,
int32_t block_count) {
return 0;
}
static void host_precompute(const cutlass::gemm::GemmCoord* host_problem_sizes_ptr,
int32_t problem_count,
int32_t block_count,
void* host_workspace_ptr) {}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename ProblemVisitor>
struct ProblemVisitorKernel {
struct SharedStorage {
typename ProblemVisitor::SharedStorage problem_visitor;
};
struct Params {
typename ProblemVisitor::Params problem_visitor_params;
int32_t* visited_problems_ptr;
int32_t* visited_tiles_ptr;
int32_t visits_per_block;
Params():
visited_problems_ptr(nullptr),
visited_tiles_ptr(nullptr),
visits_per_block(0) {}
Params(typename ProblemVisitor::Params problem_visitor_params_,
int32_t* visited_problems_ptr_,
int32_t* visited_tiles_ptr_,
int32_t visits_per_block_):
problem_visitor_params(problem_visitor_params_),
visited_problems_ptr(visited_problems_ptr_),
visited_tiles_ptr(visited_tiles_ptr_),
visits_per_block(visits_per_block_) {}
};
CUTLASS_DEVICE
void operator()(const Params& params, SharedStorage &shared_storage) {
int32_t store_offset = params.visits_per_block * blockIdx.x;
ProblemVisitor problem_visitor(params.problem_visitor_params,
shared_storage.problem_visitor,
blockIdx.x);
while (problem_visitor.next_tile()) {
int32_t problem_idx = problem_visitor.problem_index();
int32_t threadblock_idx = int32_t(problem_visitor.threadblock_idx());
if (threadIdx.x == 0) {
params.visited_problems_ptr[store_offset] = problem_idx;
params.visited_tiles_ptr[store_offset] = threadblock_idx;
++store_offset;
}
problem_visitor.advance(gridDim.x);
}
}
};
template <typename ProblemVisitor>
struct ProblemVisitorRunner {
using BaseKernel = ProblemVisitorKernel<ProblemVisitor>;
using Params = typename BaseKernel::Params;
Params params;
std::vector<cutlass::gemm::GemmCoord> host_problem_sizes;
int32_t problem_count;
int32_t threadblock_count;
int32_t visits_per_block;
cutlass::DeviceAllocation<int32_t> visited_problems;
cutlass::DeviceAllocation<int32_t> visited_tiles;
cutlass::DeviceAllocation<cutlass::gemm::GemmCoord> device_problem_sizes;
cutlass::DeviceAllocation<uint8_t> workspace;
std::vector<int32_t> host_visited_problems;
std::vector<int32_t> host_visited_tiles;
ProblemVisitorRunner(const std::vector<cutlass::gemm::GemmCoord>& host_problem_sizes_,
int32_t threadblock_count_):
host_problem_sizes(host_problem_sizes_),
problem_count(int32_t(host_problem_sizes_.size())),
threadblock_count(threadblock_count_) {}
/// Initializes GEMM state from arguments.
cutlass::Status initialize() {
size_t workspace_bytes = ProblemVisitor::get_workspace_size(
host_problem_sizes.data(),
problem_count,
threadblock_count);
workspace.reset(workspace_bytes);
std::vector<uint8_t> host_workspace(workspace_bytes);
int32_t tile_count = ProblemVisitor::group_tile_count(host_problem_sizes.data(), problem_count);
ProblemVisitor::host_precompute(host_problem_sizes.data(), problem_count,
threadblock_count, host_workspace.data());
workspace.copy_from_host(host_workspace.data(), workspace_bytes);
device_problem_sizes.reset(problem_count);
device_problem_sizes.copy_from_host(host_problem_sizes.data(), problem_count);
visits_per_block = (tile_count - 1 + threadblock_count) / threadblock_count;
int32_t total_visits = visits_per_block * threadblock_count;
visited_problems.reset(total_visits);
visited_tiles.reset(total_visits);
host_visited_problems.resize(total_visits);
host_visited_tiles.resize(total_visits);
cudaError_t result = cudaMemset(visited_problems.get(), -1, sizeof(int32_t) * total_visits);
if (result != cudaSuccess) {
return cutlass::Status::kErrorInternal;
}
result = cudaMemset(visited_tiles.get(), -1, sizeof(int32_t) * total_visits);
if (result != cudaSuccess) {
return cutlass::Status::kErrorInternal;
}
typename ProblemVisitor::Params pv_params(device_problem_sizes.get(), problem_count, workspace.get(), tile_count);
params = Params(pv_params, visited_problems.get(), visited_tiles.get(), visits_per_block);
return cutlass::Status::kSuccess;
}
bool verify() {
// Sort by problem size and then by threadblock_idx
std::vector<int32_t> indices(host_visited_problems.size());
std::iota(indices.begin(), indices.end(), 0);
std::stable_sort(indices.begin(), indices.end(),
[&](int32_t i1, int32_t i2) {
if (host_visited_problems[i1] == host_visited_problems[i2]) {
return host_visited_tiles[i1] < host_visited_tiles[i2];
}
return host_visited_problems[i1] < host_visited_problems[i2];
});
int32_t idx = 0;
// Skip any entries that were not visited
while (host_visited_problems[indices[idx]] == -1) {
++idx;
}
// Check that each problem visited has the tiles we expect
for (int32_t problem_idx = 0; problem_idx < problem_count; ++problem_idx) {
auto problem = host_problem_sizes[problem_idx];
ProblemVisitor::possibly_transpose_problem(problem);
int32_t problem_tiles = ProblemVisitor::tile_count(ProblemVisitor::grid_shape(problem));
for (int i = 0; i < problem_tiles; ++i) {
EXPECT_EQ(problem_idx, host_visited_problems[indices[idx]]);
EXPECT_EQ(i, host_visited_tiles[indices[idx]]);
++idx;
}
}
return true;
}
bool run(cudaStream_t stream = nullptr) {
cutlass::Status status = initialize();
if (status != cutlass::Status::kSuccess) {
std::cerr << "Initialization failed" << std::endl;
return false;
}
dim3 grid(threadblock_count, 1, 1);
dim3 block(ProblemVisitor::kThreadCount, 1, 1);
int smem_size = int(sizeof(typename BaseKernel::SharedStorage));
cutlass::Kernel<BaseKernel><<<grid, block, smem_size, stream>>>(params);
cudaError_t result = cudaGetLastError();
if (result != cudaSuccess) {
std::cerr << "grid launch failed with error " << cudaGetErrorString(result) << std::endl;
return false;
}
result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
std::cerr << "cudaDeviceSynchronize failed with error " << cudaGetErrorString(result) << std::endl;
return false;
}
visited_problems.copy_to_host(host_visited_problems.data());
visited_tiles.copy_to_host(host_visited_tiles.data());
return verify();
}
};
template <typename ThreadblockShape,
int PrefetchTileCount,
int ThreadCount,
bool Transpose,
cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode0,
cutlass::gemm::kernel::GroupScheduleMode... Args>
struct TestbedGroupedGemmScheduler {
using BaselinePV = BaselineProblemVisitor<cutlass::gemm::kernel::detail::GemmGroupedProblemSizeHelper<Transpose>,
ThreadblockShape,
PrefetchTileCount,
ThreadCount>;
//
// Data members
//
uint32_t seed;
int problem_count;
int threadblock_count;
std::vector<cutlass::gemm::GemmCoord> problem_sizes_host;
//
// Methods
//
TestbedGroupedGemmScheduler(uint32_t seed_ = 3080):
seed(seed_) { srand(seed); }
/// Initializes data structures
void initialize(int32_t scale_factor) {
//
// Choose random problem sizes
//
problem_sizes_host.clear();
problem_sizes_host.resize(problem_count);
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord problem(
scale_factor * (rand() % 64) + 24,
scale_factor * (rand() % 64) + 24,
scale_factor * (rand() % 64) + 24);
problem_sizes_host.at(i) = problem;
}
}
template <cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode_>
void compare_visitors(const ProblemVisitorRunner<BaselinePV>& baseline_runner) {
using PV = cutlass::gemm::kernel::GemmGroupedProblemVisitor<
ThreadblockShape,
GroupScheduleMode_,
PrefetchTileCount,
ThreadCount,
Transpose>;
ProblemVisitorRunner<PV> runner(problem_sizes_host, threadblock_count);
EXPECT_TRUE(runner.run());
// Check that this problem visitor visits the same problems and tiles as the baseline
EXPECT_EQ(baseline_runner.host_visited_problems, runner.host_visited_problems);
EXPECT_EQ(baseline_runner.host_visited_tiles, runner.host_visited_tiles);
}
template <cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode1_,
cutlass::gemm::kernel::GroupScheduleMode GroupScheduleMode2_,
cutlass::gemm::kernel::GroupScheduleMode... Rest>
void compare_visitors(const ProblemVisitorRunner<BaselinePV>& baseline_runner) {
// Compare the next visitor with the baseline visitor
compare_visitors<GroupScheduleMode1_>(baseline_runner);
// Recurse to compare the next visitors
compare_visitors<GroupScheduleMode2_, Rest...>(baseline_runner);
}
/// Executes the test on all scheduler modes
void run(int problem_count, int threadblock_count, int scale_factor=8) {
this->problem_count = problem_count;
this->threadblock_count = threadblock_count;
// Initialize the problem
initialize(scale_factor);
// Run the baseline visitor to which we will compare all other visitors
ProblemVisitorRunner<BaselinePV> baseline_runner(problem_sizes_host, threadblock_count);
EXPECT_TRUE(baseline_runner.run());
compare_visitors<Args...>(baseline_runner);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // device
} // gemm
} // test
/////////////////////////////////////////////////////////////////////////////////////////////////